1
0
Fork 0
mirror of https://git.rwth-aachen.de/acs/public/villas/node/ synced 2025-03-09 00:00:00 +01:00

move checks into *_check(), set default values in _init()

This commit is contained in:
Steffen Vogel 2017-03-27 12:57:41 +02:00
parent 6f4c139c2f
commit 644352538d
6 changed files with 191 additions and 117 deletions

View file

@ -26,8 +26,8 @@ struct fpga_card {
enum state state; /**< The state of this FPGA card. */
struct pci_dev filter; /**< Filter for PCI device. */
struct vfio_dev vd; /**< VFIO device handle. */
struct pci_device filter; /**< Filter for PCI device. */
struct vfio_device vfio_device; /**< VFIO device handle. */
int do_reset; /**< Reset VILLASfpga during startup? */
int affinity; /**< Affinity for MSI interrupts */

View file

@ -14,7 +14,7 @@
#define PCI_SLOT(devfn) (((devfn) >> 3) & 0x1f)
#define PCI_FUNC(devfn) ((devfn) & 0x07)
struct pci_dev {
struct pci_device {
struct {
int vendor;
int device;
@ -30,7 +30,7 @@ struct pci_dev {
};
struct pci {
struct list devices; /**< List of available PCI devices in the system (struct pci_dev) */
struct list devices; /**< List of available PCI devices in the system (struct pci_device) */
};
/** Initialize Linux PCI handle.
@ -45,18 +45,18 @@ int pci_init(struct pci *p);
/** Destroy handle. */
int pci_destroy(struct pci *p);
int pci_dev_parse_slot(struct pci_dev *f, const char *str, const char **error);
int pci_device_parse_slot(struct pci_device *f, const char *str, const char **error);
int pci_dev_parse_id(struct pci_dev *f, const char *str, const char **error);
int pci_device_parse_id(struct pci_device *f, const char *str, const char **error);
int pci_dev_compare(const struct pci_dev *d, const struct pci_dev *f);
int pci_device_compare(const struct pci_device *d, const struct pci_device *f);
struct pci_dev * pci_lookup_device(struct pci *p, struct pci_dev *filter);
struct pci_device * pci_lookup_device(struct pci *p, struct pci_device *filter);
/** Bind a new LKM to the PCI device */
int pci_attach_driver(struct pci_dev *d, const char *driver);
int pci_attach_driver(struct pci_device *d, const char *driver);
/** Return the IOMMU group of this PCI device or -1 if the device is not in a group. */
int pci_get_iommu_group(struct pci_dev *d);
int pci_get_iommu_group(struct pci_device *d);
/** @} */

View file

@ -21,20 +21,20 @@
#define VFIO_DEV(x) "/dev/vfio/" x
/* Forward declarations */
struct pci_dev;
struct pci_device;
struct vfio_group {
int fd; /**< VFIO group file descriptor */
int index; /**< Index of the IOMMU group as listed under /sys/kernel/iommu_groups/ */
int fd; /**< VFIO group file descriptor */
int index; /**< Index of the IOMMU group as listed under /sys/kernel/iommu_groups/ */
struct vfio_group_status status; /**< Status of group */
struct vfio_group_status status; /**< Status of group */
struct list devices;
struct vfio_container *container; /**< The VFIO container to which this group is belonging */
struct vfio_container *container; /**< The VFIO container to which this group is belonging */
};
struct vfio_dev {
struct vfio_device {
char *name; /**< Name of the device as listed under /sys/kernel/iommu_groups/[vfio_group::index]/devices/ */
int fd; /**< VFIO device file descriptor */
@ -44,7 +44,7 @@ struct vfio_dev {
void **mappings;
struct pci_dev *pdev; /**< libpci handle of the device */
struct pci_device *pci_device; /**< libpci handle of the device */
struct vfio_group *group; /**< The VFIO group this device belongs to */
};
@ -65,25 +65,25 @@ int vfio_init(struct vfio_container *c);
int vfio_group_attach(struct vfio_group *g, struct vfio_container *c, int index);
/** Initialize a VFIO device, lookup the VFIO group it belongs to, create the group if not already existing. */
int vfio_dev_attach(struct vfio_dev *d, struct vfio_container *c, const char *name, int index);
int vfio_device_attach(struct vfio_device *d, struct vfio_container *c, const char *name, int index);
/** Initialie a VFIO-PCI device (uses vfio_dev_attach() internally) */
int vfio_pci_attach(struct vfio_dev *d, struct vfio_container *c, struct pci_dev *pdev);
/** Initialie a VFIO-PCI device (uses vfio_device_attach() internally) */
int vfio_pci_attach(struct vfio_device *d, struct vfio_container *c, struct pci_device *pdev);
/** Hot resets a VFIO-PCI device */
int vfio_pci_reset(struct vfio_dev *d);
int vfio_pci_reset(struct vfio_device *d);
int vfio_pci_msi_init(struct vfio_dev *d, int efds[32]);
int vfio_pci_msi_init(struct vfio_device *d, int efds[32]);
int vfio_pci_msi_deinit(struct vfio_dev *d, int efds[32]);
int vfio_pci_msi_deinit(struct vfio_device *d, int efds[32]);
int vfio_pci_msi_find(struct vfio_dev *d, int nos[32]);
int vfio_pci_msi_find(struct vfio_device *d, int nos[32]);
/** Enable memory accesses and bus mastering for PCI device */
int vfio_pci_enable(struct vfio_dev *d);
int vfio_pci_enable(struct vfio_device *d);
/** Reset a VFIO device */
int vfio_dev_reset(struct vfio_dev *d);
int vfio_device_reset(struct vfio_device *d);
/** Release memory and close container */
int vfio_destroy(struct vfio_container *c);
@ -92,13 +92,13 @@ int vfio_destroy(struct vfio_container *c);
int vfio_group_destroy(struct vfio_group *g);
/** Release memory of device */
int vfio_dev_destroy(struct vfio_dev *g);
int vfio_device_destroy(struct vfio_device *g);
/** Print a dump of all attached groups and their devices including regions and IRQs */
void vfio_dump(struct vfio_container *c);
/** Map a device memory region to the application address space (e.g. PCI BARs) */
void * vfio_map_region(struct vfio_dev *d, int idx);
void * vfio_map_region(struct vfio_device *d, int idx);
/** Map VM to an IOVA, which is accessible by devices in the container */
int vfio_map_dma(struct vfio_container *c, uint64_t virt, uint64_t phys, size_t len);
@ -107,6 +107,6 @@ int vfio_map_dma(struct vfio_container *c, uint64_t virt, uint64_t phys, size_t
int vfio_unmap_dma(struct vfio_container *c, uint64_t virt, uint64_t phys, size_t len);
/** munmap() a region which has been mapped by vfio_map_region() */
int vfio_unmap_region(struct vfio_dev *d, int idx);
int vfio_unmap_region(struct vfio_device *d, int idx);
/** @} */

View file

@ -19,52 +19,21 @@
int fpga_card_init(struct fpga_card *c, struct pci *pci, struct vfio_container *vc)
{
int ret;
assert(c->state = STATE_DESTROYED);
struct pci_dev *pdev;
fpga_card_check(c);
c->vfio_container = vc;
c->pci = pci;
assert(c->state != STATE_DESTROYED);
/* Search for FPGA card */
pdev = pci_lookup_device(pci, &c->filter);
if (!pdev)
error("Failed to find PCI device");
/* Attach PCIe card to VFIO container */
ret = vfio_pci_attach(&c->vd, vc, pdev);
if (ret)
error("Failed to attach VFIO device");
/* Map PCIe BAR */
c->map = vfio_map_region(&c->vd, VFIO_PCI_BAR0_REGION_INDEX);
if (c->map == MAP_FAILED)
serror("Failed to mmap() BAR0");
/* Enable memory access and PCI bus mastering for DMA */
ret = vfio_pci_enable(&c->vd);
if (ret)
serror("Failed to enable PCI device");
list_init(&c->ips);
/* Reset system? */
if (c->do_reset) {
/* Reset / detect PCI device */
ret = vfio_pci_reset(&c->vd);
if (ret)
serror("Failed to reset PCI device");
ret = fpga_card_reset(c);
if (ret)
error("Failed to reset FGPA card");
}
/* Initialize IP cores */
list_foreach(struct fpga_ip *i, &c->ips) {
ret = fpga_ip_init(i);
if (ret)
error("Failed to initalize IP core: %s (%u)", i->name, ret);
}
/* Default values */
c->filter.id.vendor = FPGA_PCI_VID_XILINX;
c->filter.id.device = FPGA_PCI_PID_VFPGA;
c->affinity = 0;
c->do_reset = 0;
c->state = STATE_INITIALIZED;
return 0;
}
@ -75,23 +44,16 @@ int fpga_card_parse(struct fpga_card *c, config_setting_t *cfg)
const char *slot, *id, *err;
config_setting_t *cfg_ips, *cfg_slot, *cfg_id;
/* Default values */
c->filter.id.vendor = FPGA_PCI_VID_XILINX;
c->filter.id.device = FPGA_PCI_PID_VFPGA;
c->name = config_setting_name(cfg);
if (!config_setting_lookup_int(cfg, "affinity", &c->affinity))
c->affinity = 0;
if (!config_setting_lookup_bool(cfg, "do_reset", &c->do_reset))
c->do_reset = 0;
config_setting_lookup_int(cfg, "affinity", &c->affinity);
config_setting_lookup_bool(cfg, "do_reset", &c->do_reset);
cfg_slot = config_setting_get_member(cfg, "slot");
if (cfg_slot) {
slot = config_setting_get_string(cfg_slot);
if (slot) {
ret = pci_dev_parse_slot(&c->filter, slot, &err);
ret = pci_device_parse_slot(&c->filter, slot, &err);
if (ret)
cerror(cfg_slot, "Failed to parse PCI slot: %s", err);
}
@ -103,7 +65,7 @@ int fpga_card_parse(struct fpga_card *c, config_setting_t *cfg)
if (cfg_id) {
id = config_setting_get_string(cfg_id);
if (id) {
ret = pci_dev_parse_id(&c->filter, (char*) id, &err);
ret = pci_device_parse_id(&c->filter, (char*) id, &err);
if (ret)
cerror(cfg_id, "Failed to parse PCI id: %s", err);
}
@ -118,13 +80,28 @@ int fpga_card_parse(struct fpga_card *c, config_setting_t *cfg)
for (int i = 0; i < config_setting_length(cfg_ips); i++) {
config_setting_t *cfg_ip = config_setting_get_elem(cfg_ips, i);
const char *vlnv;
struct fpga_ip_type *vt;
struct fpga_ip ip = {
.card = c
.card = c,
.state = STATE_DESTROYED
};
if (!config_setting_lookup_string(cfg, "vlnv", &vlnv))
cerror(cfg, "FPGA IP core %s is missing the VLNV identifier", c->name);
vt = fpga_ip_type_lookup(vlnv);
if (!vt)
cerror(cfg, "FPGA IP core VLNV identifier '%s' is invalid", vlnv);
ret = fpga_ip_init(&ip, vt);
if (ret)
error("Failed to initalize FPGA IP core");
ret = fpga_ip_parse(&ip, cfg_ip);
if (ret)
cerror(cfg_ip, "Failed to parse VILLASfpga IP core");
cerror(cfg_ip, "Failed to parse FPGA IP core");
list_push(&c->ips, memdup(&ip, sizeof(ip)));
}
@ -135,24 +112,121 @@ int fpga_card_parse(struct fpga_card *c, config_setting_t *cfg)
return 0;
}
int fpga_card_parse_list(struct list *cards, config_setting_t *cfg)
{
int ret;
if (!config_setting_is_group(cfg))
cerror(cfg, "FPGA configuration section must be a group");
for (int i = 0; i < config_setting_length(cfg); i++) {
config_setting_t *cfg_fpga = config_setting_get_elem(cfg, i);
struct fpga_card c;
ret = fpga_card_parse(&c, cfg_fpga);
if (ret)
cerror(cfg_fpga, "Failed to parse FPGA card configuration");
list_push(cards, memdup(&c, sizeof(c)));
}
return 0;
}
int fpga_card_start(struct fpga_card *c)
{
int ret;
struct pci_device *pdev;
assert(c->state == STATE_CHECKED);
/* Search for FPGA card */
pdev = pci_lookup_device(c->pci, &c->filter);
if (!pdev)
error("Failed to find PCI device");
/* Attach PCIe card to VFIO container */
ret = vfio_pci_attach(&c->vfio_device, c->vfio_container, pdev);
if (ret)
error("Failed to attach VFIO device");
/* Map PCIe BAR */
c->map = vfio_map_region(&c->vfio_device, VFIO_PCI_BAR0_REGION_INDEX);
if (c->map == MAP_FAILED)
serror("Failed to mmap() BAR0");
/* Enable memory access and PCI bus mastering for DMA */
ret = vfio_pci_enable(&c->vfio_device);
if (ret)
serror("Failed to enable PCI device");
/* Reset system? */
if (c->do_reset) {
/* Reset / detect PCI device */
ret = vfio_pci_reset(&c->vfio_device);
if (ret)
serror("Failed to reset PCI device");
ret = fpga_card_reset(c);
if (ret)
error("Failed to reset FGPA card");
}
/* Initialize IP cores */
for (size_t j = 0; j < list_length(&c->ips); j++) {
struct fpga_ip *i = list_at(&c->ips, j);
ret = fpga_ip_start(i);
if (ret)
error("Failed to initalize FPGA IP core: %s (%u)", i->name, ret);
}
c->state = STATE_STARTED;
return 0;
}
int fpga_card_stop(struct fpga_card *c)
{
int ret;
assert(c->state == STATE_STOPPED);
for (size_t j = 0; j < list_length(&c->ips); j++) {
struct fpga_ip *i = list_at(&c->ips, j);
ret = fpga_ip_stop(i);
if (ret)
error("Failed to stop FPGA IP core: %s (%u)", i->name, ret);
}
c->state = STATE_STOPPED;
return 0;
}
void fpga_card_dump(struct fpga_card *c)
{
info("VILLASfpga card:");
{ INDENT
info("Slot: %04x:%02x:%02x.%d", c->vd.pdev->slot.domain, c->vd.pdev->slot.bus, c->vd.pdev->slot.device, c->vd.pdev->slot.function);
info("Vendor ID: %04x", c->vd.pdev->id.vendor);
info("Device ID: %04x", c->vd.pdev->id.device);
info("Class ID: %04x", c->vd.pdev->id.class);
info("Slot: %04x:%02x:%02x.%d", c->vfio_device.pci_device->slot.domain, c->vfio_device.pci_device->slot.bus, c->vfio_device.pci_device->slot.device, c->vfio_device.pci_device->slot.function);
info("Vendor ID: %04x", c->vfio_device.pci_device->id.vendor);
info("Device ID: %04x", c->vfio_device.pci_device->id.device);
info("Class ID: %04x", c->vfio_device.pci_device->id.class);
info("BAR0 mapped at %p", c->map);
info("IP blocks:");
list_foreach(struct fpga_ip *i, &c->ips) { INDENT
for (size_t j = 0; j < list_length(&c->ips); j++) { INDENT
struct fpga_ip *i = list_at(&c->ips, j);
fpga_ip_dump(i);
}
}
vfio_dump(c->vd.group->container);
vfio_dump(c->vfio_device.group->container);
}
int fpga_card_check(struct fpga_card *c)
@ -176,7 +250,7 @@ int fpga_card_check(struct fpga_card *c)
int fpga_card_destroy(struct fpga_card *c)
{
list_destroy(&c->ips, (dtor_cb_t) fpga_ip_destroy, true);
return 0;
}
@ -186,7 +260,7 @@ int fpga_card_reset(struct fpga_card *c)
char state[4096];
/* Save current state of PCI configuration space */
ret = pread(c->vd.fd, state, sizeof(state), (off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40);
ret = pread(c->vfio_device.fd, state, sizeof(state), (off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40);
if (ret != sizeof(state))
return -1;
@ -198,7 +272,7 @@ int fpga_card_reset(struct fpga_card *c)
usleep(100000);
/* Restore previous state of PCI configuration space */
ret = pwrite(c->vd.fd, state, sizeof(state), (off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40);
ret = pwrite(c->vfio_device.fd, state, sizeof(state), (off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40);
if (ret != sizeof(state))
return -1;

View file

@ -32,7 +32,7 @@ int pci_init(struct pci *p)
}
while ((entry = readdir(dp))) {
struct pci_dev d;
struct pci_device d;
struct { const char *s; int *p; } map[] = {
{ "vendor", &d.id.vendor },
@ -74,7 +74,7 @@ int pci_destroy(struct pci *p)
return 0;
}
int pci_dev_parse_slot(struct pci_dev *f, const char *s, const char **error)
int pci_device_parse_slot(struct pci_device *f, const char *s, const char **error)
{
char *str = strdup(s);
char *colon = strrchr(str, ':');
@ -149,7 +149,7 @@ fail:
}
/* ID filter syntax: [vendor]:[device][:class] */
int pci_dev_parse_id(struct pci_dev *f, const char *str, const char **error)
int pci_device_parse_id(struct pci_device *f, const char *str, const char **error)
{
char *s, *c, *e;
@ -205,7 +205,7 @@ fail:
return -1;
}
int pci_dev_compare(const struct pci_dev *d, const struct pci_dev *f)
int pci_device_compare(const struct pci_device *d, const struct pci_device *f)
{
if ((f->slot.domain >= 0 && f->slot.domain != d->slot.domain) ||
(f->slot.bus >= 0 && f->slot.bus != d->slot.bus) ||
@ -226,12 +226,12 @@ int pci_dev_compare(const struct pci_dev *d, const struct pci_dev *f)
return 1;
}
struct pci_dev * pci_lookup_device(struct pci *p, struct pci_dev *f)
struct pci_device * pci_lookup_device(struct pci *p, struct pci_device *f)
{
return list_search(&p->devices, (cmp_cb_t) pci_dev_compare, (void *) f);
return list_search(&p->devices, (cmp_cb_t) pci_device_compare, (void *) f);
}
int pci_attach_driver(struct pci_dev *d, const char *driver)
int pci_attach_driver(struct pci_device *d, const char *driver)
{
FILE *f;
char fn[256];
@ -259,7 +259,7 @@ int pci_attach_driver(struct pci_dev *d, const char *driver)
return 0;
}
int pci_get_iommu_group(struct pci_dev *d)
int pci_get_iommu_group(struct pci_device *d)
{
int ret;
char *group, link[1024], sysfs[1024];

View file

@ -90,7 +90,7 @@ int vfio_group_destroy(struct vfio_group *g)
{
int ret;
list_destroy(&g->devices, (dtor_cb_t) vfio_dev_destroy, false);
list_destroy(&g->devices, (dtor_cb_t) vfio_device_destroy, false);
ret = ioctl(g->fd, VFIO_GROUP_UNSET_CONTAINER);
if (ret)
@ -107,7 +107,7 @@ int vfio_group_destroy(struct vfio_group *g)
return 0;
}
int vfio_dev_destroy(struct vfio_dev *d)
int vfio_device_destroy(struct vfio_device *d)
{
int ret;
@ -204,7 +204,7 @@ int vfio_group_attach(struct vfio_group *g, struct vfio_container *c, int index)
return 0;
}
int vfio_pci_attach(struct vfio_dev *d, struct vfio_container *c, struct pci_dev *pdev)
int vfio_pci_attach(struct vfio_device *d, struct vfio_container *c, struct pci_device *pdev)
{
char name[32];
int ret;
@ -226,22 +226,22 @@ int vfio_pci_attach(struct vfio_dev *d, struct vfio_container *c, struct pci_dev
/* VFIO device name consists of PCI BDF */
snprintf(name, sizeof(name), "%04x:%02x:%02x.%x", pdev->slot.domain, pdev->slot.bus, pdev->slot.device, pdev->slot.function);
ret = vfio_dev_attach(d, c, name, index);
ret = vfio_device_attach(d, c, name, index);
if (ret < 0)
return ret;
/* Check if this is really a vfio-pci device */
if (!(d->info.flags & VFIO_DEVICE_FLAGS_PCI)) {
vfio_dev_destroy(d);
vfio_device_destroy(d);
return -1;
}
d->pdev = pdev;
d->pci_device = pdev;
return 0;
}
int vfio_dev_attach(struct vfio_dev *d, struct vfio_container *c, const char *name, int index)
int vfio_device_attach(struct vfio_device *d, struct vfio_container *c, const char *name, int index)
{
int ret;
struct vfio_group *g = NULL;
@ -313,7 +313,7 @@ int vfio_dev_attach(struct vfio_dev *d, struct vfio_container *c, const char *na
return 0;
}
int vfio_pci_reset(struct vfio_dev *d)
int vfio_pci_reset(struct vfio_device *d)
{
int ret;
@ -353,7 +353,7 @@ int vfio_pci_reset(struct vfio_dev *d)
return ret;
}
int vfio_pci_msi_find(struct vfio_dev *d, int nos[32])
int vfio_pci_msi_find(struct vfio_device *d, int nos[32])
{
int ret, idx, irq;
char *end, *col, *last, line[1024], name[13];
@ -393,7 +393,7 @@ int vfio_pci_msi_find(struct vfio_dev *d, int nos[32])
return 0;
}
int vfio_pci_msi_deinit(struct vfio_dev *d, int efds[32])
int vfio_pci_msi_deinit(struct vfio_device *d, int efds[32])
{
int ret, irq_setlen, irq_count = d->irqs[VFIO_PCI_MSI_IRQ_INDEX].count;
struct vfio_irq_set *irq_set;
@ -427,7 +427,7 @@ int vfio_pci_msi_deinit(struct vfio_dev *d, int efds[32])
return irq_count;
}
int vfio_pci_msi_init(struct vfio_dev *d, int efds[32])
int vfio_pci_msi_init(struct vfio_device *d, int efds[32])
{
int ret, irq_setlen, irq_count = d->irqs[VFIO_PCI_MSI_IRQ_INDEX].count;
struct vfio_irq_set *irq_set;
@ -462,7 +462,7 @@ int vfio_pci_msi_init(struct vfio_dev *d, int efds[32])
return irq_count;
}
int vfio_pci_enable(struct vfio_dev *d)
int vfio_pci_enable(struct vfio_device *d)
{
int ret;
uint32_t reg;
@ -486,7 +486,7 @@ int vfio_pci_enable(struct vfio_dev *d)
return 0;
}
int vfio_dev_reset(struct vfio_dev *d)
int vfio_device_reset(struct vfio_device *d)
{
if (d->info.flags & VFIO_DEVICE_FLAGS_RESET)
return ioctl(d->fd, VFIO_DEVICE_RESET);
@ -543,7 +543,7 @@ void vfio_dump(struct vfio_container *v)
}
}
void * vfio_map_region(struct vfio_dev *d, int idx)
void * vfio_map_region(struct vfio_device *d, int idx)
{
struct vfio_region_info *r = &d->regions[idx];
@ -555,7 +555,7 @@ void * vfio_map_region(struct vfio_dev *d, int idx)
return d->mappings[idx];
}
int vfio_unmap_region(struct vfio_dev *d, int idx)
int vfio_unmap_region(struct vfio_device *d, int idx)
{
int ret;
struct vfio_region_info *r = &d->regions[idx];