Mirrors/VILLASnode
1
0
Fork 0
mirror of https://git.rwth-aachen.de/acs/public/villas/node/ synced 2025-03-09 00:00:00 +01:00
Code Issues Releases Wiki Activity

vfio: increase logging verbosity

Browse source 
Signed-off-by: Niklas Eiling <niklas.eiling@eonerc.rwth-aachen.de>
This commit is contained in:
Niklas Eiling 2022-12-05 12:09:25 +01:00 • committed by Steffen Vogel
parent e68b9d4e64
commit c35bab4490
6 changed files with 76 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
common/include/villas/kernel/vfio_container.hpp
View file

@ -24,6 +24,8 @@ namespace villas {
namespace kernel {
namespace vfio {
static constexpr size_t EXTENSION_SIZE = VFIO_UPDATE_VADDR+1;
class Container {
public:
Container();
@ -64,7 +66,8 @@ private:
int fd;
int version;
int extensions;
std::array<bool, EXTENSION_SIZE> extensions;
uint64_t iova_next; /**< Next free IOVA address */
bool hasIommu;

6
common/include/villas/kernel/vfio_device.hpp
View file

@ -22,12 +22,6 @@
#include <villas/kernel/pci.hpp>
#include <villas/log.hpp>
#define VFIO_PATH "/dev/vfio/"
#define VFIO_DEV VFIO_PATH "vfio"
#ifndef VFIO_NOIOMMU_IOMMU
#define VFIO_NOIOMMU_IOMMU 8
#endif
namespace villas {
namespace kernel {

3
common/include/villas/kernel/vfio_group.hpp
View file

@ -25,6 +25,9 @@ namespace villas {
namespace kernel {
namespace vfio {
#define VFIO_PATH "/dev/vfio/"
#define VFIO_DEV VFIO_PATH "vfio"
class Group {
public:
Group(int index, bool iommuEnabled);

72
common/lib/kernel/vfio_container.cpp
View file

@ -18,6 +18,7 @@
#include <string>
#include <sstream>
#include <limits>
#include <array>
#include <cstdlib>
#include <cstring>
@ -36,11 +37,33 @@
using namespace villas::kernel::vfio;
#ifndef VFIO_NOIOMMU_IOMMU
#define VFIO_NOIOMMU_IOMMU 8
#endif
static std::array<std::string, EXTENSION_SIZE> construct_vfio_extension_str() {
std::array<std::string, EXTENSION_SIZE> ret;
ret[VFIO_TYPE1_IOMMU] = "Type 1";
ret[VFIO_SPAPR_TCE_IOMMU] = "SPAPR TCE";
ret[VFIO_TYPE1v2_IOMMU] = "Type 1 v2";
ret[VFIO_DMA_CC_IOMMU] = "DMA CC";
ret[VFIO_EEH] = "EEH";
ret[VFIO_TYPE1_NESTING_IOMMU] = "Type 1 Nesting";
ret[VFIO_SPAPR_TCE_v2_IOMMU] = "SPAPR TCE v2";
ret[VFIO_NOIOMMU_IOMMU] = "No IOMMU";
ret[VFIO_UNMAP_ALL] = "Unmap all";
ret[VFIO_UPDATE_VADDR] = "Update vaddr";
return ret;
}
static std::array<std::string, EXTENSION_SIZE> VFIO_EXTENSION_STR = construct_vfio_extension_str();
Container::Container() :
fd(-1),
version(0),
extensions(0),
extensions(),
iova_next(0),
hasIommu(false),
groups(),
@ -51,12 +74,13 @@ Container::Container() :
};
for (const char* module : requiredKernelModules) {
if (kernel::loadModule(module) != 0)
if (kernel::loadModule(module) != 0) {
throw RuntimeError("Kernel module '{}' required but could not be loaded. "
"Please load manually!", module);
}
}
// Open VFIO API
// Create a VFIO Container
fd = open(VFIO_DEV, O_RDWR);
if (fd < 0)
throw RuntimeError("Failed to open VFIO container");
@ -64,29 +88,30 @@ Container::Container() :
// Check VFIO API version
version = ioctl(fd, VFIO_GET_API_VERSION);
if (version < 0 || version != VFIO_API_VERSION)
throw RuntimeError("Failed to get VFIO version");
throw RuntimeError("Unknown API version: {}", version);
// Check available VFIO extensions (IOMMU types)
extensions = 0;
for (unsigned int i = VFIO_TYPE1_IOMMU; i <= VFIO_NOIOMMU_IOMMU; i++) {
for (unsigned int i = VFIO_TYPE1_IOMMU; i < EXTENSION_SIZE; i++) {
int ret = ioctl(fd, VFIO_CHECK_EXTENSION, i);
if (ret < 0)
throw RuntimeError("Failed to get VFIO extensions");
else if (ret > 0)
extensions |= (1 << i);
if (ret == 0)
extensions[i] = false;
else {
extensions[i] = true;
}
log->debug("VFIO extension {} is {} ({})", i, extensions[i] ? "available" : "not available", VFIO_EXTENSION_STR[i]);
}
hasIommu = false;
if (not (extensions & (1 << VFIO_NOIOMMU_IOMMU))) {
if (not (extensions & (1 << VFIO_TYPE1_IOMMU)))
throw RuntimeError("No supported IOMMU extension found");
else
hasIommu = true;
if (extensions[VFIO_TYPE1_IOMMU]) {
log->debug("Using VFIO type {} ({})", VFIO_TYPE1_IOMMU, VFIO_EXTENSION_STR[VFIO_TYPE1_IOMMU]);
hasIommu = true;
} else if (extensions[VFIO_NOIOMMU_IOMMU]) {
log->debug("Using VFIO type {} ({})", VFIO_NOIOMMU_IOMMU, VFIO_EXTENSION_STR[VFIO_NOIOMMU_IOMMU]);
hasIommu = false;
} else {
throw RuntimeError("No supported IOMMU type available");
}
log->debug("Version: {:#x}", version);
log->debug("Extensions: {:#x}", extensions);
log->debug("IOMMU: {}", hasIommu ? "yes" : "no");
}
@ -127,10 +152,7 @@ void Container::attachGroup(std::shared_ptr<Group> group)
group->setAttachedToContainer();
if (!group->checkStatus())
throw RuntimeError("bad VFIO group status for group {}.", group->getIndex());
else
log->debug("Attached new group {} to VFIO container", group->getIndex());
log->debug("Attached new group {} to VFIO container with fd {}", group->getIndex(), fd);
// Push to our list
groups.push_back(std::move(group));
@ -157,7 +179,11 @@ void Container::dump()
{
log->info("File descriptor: {}", fd);
log->info("Version: {}", version);
log->info("Extensions: 0x{:x}", extensions);
// Check available VFIO extensions (IOMMU types)
for (size_t i = 0; i < extensions.size(); i++) {
log->debug("VFIO extension {} ({}) is {}", extensions[i], VFIO_EXTENSION_STR[i], extensions[i] ? "available" : "not available");
}
for (auto &group : groups) {
group->dump();

21
common/lib/kernel/vfio_device.cpp
View file

@ -79,8 +79,13 @@ Device::Device(const std::string &name, int groupFileDescriptor, const kernel::p
if (ret < 0)
throw RuntimeError("Failed to get VFIO device info for: {}", name);
log->debug("Device has {} regions", info.num_regions);
log->debug("Device has {} IRQs", info.num_irqs);
log->debug("device info: flags: 0x{:x}, num_regions: {}, num_irqs: {}, cap_offset: 0x{:x}",
info.flags, info.num_regions, info.num_irqs, info.cap_offset);
// device_info.num_region reports always 9 and includes a VGA region, which is only supported on
// certain device IDs. So for non-VGA devices VFIO_PCI_CONFIG_REGION_INDEX will be the highest
// region index. This is the config space.
info.num_regions = VFIO_PCI_CONFIG_REGION_INDEX + 1;
// Reserve slots already so that we can use the []-operator for access
irqs.resize(info.num_irqs);
@ -97,7 +102,10 @@ Device::Device(const std::string &name, int groupFileDescriptor, const kernel::p
ret = ioctl(fd, VFIO_DEVICE_GET_REGION_INFO, &region);
if (ret < 0)
throw RuntimeError("Failed to get region of VFIO device: {}", name);
throw RuntimeError("Failed to get region {} of VFIO device: {}", i, name);
log->debug("region {} info: flags: 0x{:x}, cap_offset: 0x{:x}, size: 0x{:x}, offset: 0x{:x}",
region.index, region.flags, region.cap_offset, region.size, region.offset);
regions[i] = region;
}
@ -113,7 +121,12 @@ Device::Device(const std::string &name, int groupFileDescriptor, const kernel::p
ret = ioctl(fd, VFIO_DEVICE_GET_IRQ_INFO, &irq);
if (ret < 0)
throw RuntimeError("Failed to get IRQs of VFIO device: {}", name);
throw RuntimeError("Failed to get IRQ {} of VFIO device: {}", i, name);
log->debug("irq {} info: flags: 0x{:x}, count: {}",
irq.index, irq.flags, irq.count);
irqs[i] = irq;
}

7
common/lib/kernel/vfio_group.cpp
View file

@ -58,6 +58,8 @@ Group::Group(int index, bool iommuEnabled) :
log->debug("VFIO group {} (fd {}) has path {}",
index, fd, groupPath.str());
checkStatus();
}
std::shared_ptr<Device> Group::attachDevice(std::shared_ptr<Device> device)
@ -81,10 +83,6 @@ std::shared_ptr<Device> Group::attachDevice(const std::string& name, const kerne
bool Group::checkStatus()
{
int ret;
if (!attachedToContainer) {
log->error("Group {} is not attached to a container", index);
return false;
}
// Check group viability and features
status.argsz = sizeof(status);
@ -99,6 +97,7 @@ bool Group::checkStatus()
log->error("VFIO group is not available: bind all devices to the VFIO driver!");
return false;
}
log->debug("VFIO group is {} viable", index);
return true;
}