diff --git a/include/villas/kernel/pci.h b/include/villas/kernel/pci.h index 9425f4690..747f5c488 100644 --- a/include/villas/kernel/pci.h +++ b/include/villas/kernel/pci.h @@ -10,16 +10,58 @@ #ifndef _PCI_H_ #define _PCI_H_ -#include +#include "list.h" -struct pci_access * pci_get_handle(); +#define PCI_SLOT(devfn) (((devfn) >> 3) & 0x1f) +#define PCI_FUNC(devfn) ((devfn) & 0x07) -void pci_release_handle(); +struct pci_dev { + struct { + int vendor; + int device; + int class; + } id; + + struct { + int domain; + int bus; + int device; + int function; + } slot; /**< Bus, Device, Function (BDF) */ +}; -struct pci_dev * pci_find_device(struct pci_access *pacc, struct pci_filter *f); +struct pci { + struct list devices; /**> List of available PCI devices in the system (struct pci_dev) */ +}; +/** Initialize Linux PCI handle. + * + * This search for all available PCI devices under /sys/bus/pci + * + * @retval 0 Success. Everything went well. + * @retval <0 Error. Something went wrong. + */ +int pci_init(struct pci *p); + +/** Destroy handle. */ +void pci_destroy(struct pci *p); + +int pci_dev_init(struct pci_dev *d); + +void pci_dev_destroy(struct pci_dev *d); + +int pci_dev_parse_slot(struct pci_dev *f, const char *str, const char **error); + +int pci_dev_parse_id(struct pci_dev *f, const char *str, const char **error); + +int pci_dev_compare(const struct pci_dev *d, const struct pci_dev *f); + +struct pci_dev * pci_lookup_device(struct pci *p, struct pci_dev *filter); + +/** Bind a new LKM to the PCI device */ int pci_attach_driver(struct pci_dev *d, const char *driver); -int pci_get_iommu_group(struct pci_dev *pdev); +/** Return the IOMMU group of this PCI device or -1 if the device is not in a group. */ +int pci_get_iommu_group(struct pci_dev *d); #endif /* _PCI_H_ */ \ No newline at end of file diff --git a/include/villas/kernel/vfio.h b/include/villas/kernel/vfio.h index 240b34971..7f1a227aa 100644 --- a/include/villas/kernel/vfio.h +++ b/include/villas/kernel/vfio.h @@ -11,7 +11,6 @@ #define _VFIO_H_ #include -#include #include #include @@ -23,6 +22,7 @@ /* Forward declaration */ struct dma_mem; +struct pci_dev; struct vfio_group { int fd; /**< VFIO group file descriptor */ diff --git a/include/villas/nodes/fpga.h b/include/villas/nodes/fpga.h index 411eac847..0dadcfd94 100644 --- a/include/villas/nodes/fpga.h +++ b/include/villas/nodes/fpga.h @@ -18,8 +18,7 @@ #define _FPGA_H_ #include "kernel/vfio.h" - -#include +#include "kernel/pci.h" #include "fpga/dma.h" #include "fpga/ip.h" @@ -29,7 +28,7 @@ #include "list.h" struct fpga { - struct pci_filter filter; /**< Filter for libpci with device id & slot */ + struct pci_dev filter; /**< Filter for PCI device. */ struct vfio_dev vd; /**< VFIO device handle. */ int do_reset; /**< Reset VILLASfpga during startup? */ diff --git a/lib/kernel/pci.c b/lib/kernel/pci.c index 5a80b05be..7ed1ba71c 100644 --- a/lib/kernel/pci.c +++ b/lib/kernel/pci.c @@ -6,71 +6,236 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. **********************************************************************************/ -#include -#include -#include -#include +#include #include +#include +#include #include "log.h" #include "kernel/pci.h" #include "config.h" -static struct pci_access *pacc; - -static void pci_log(char *msg, ...) +int pci_init(struct pci *p) { - char *tmp = strdup(msg); + struct dirent *entry; + DIR *dp; + FILE *f; + char path[256]; + int ret; + + snprintf(path, sizeof(path), "%s/bus/pci/devices", SYSFS_PATH); - va_list ap; - va_start(ap, msg); - log_vprint("PCI ", strtok(tmp, "\n"), ap); - va_end(ap); - free(tmp); -} - -struct pci_access * pci_get_handle() -{ - if (pacc) - return pacc; /* Singleton */ - - pacc = pci_alloc(); /* Get the pci_access structure */ - if (!pacc) - error("Failed to allocate PCI access structure"); - - pci_init(pacc); /* Initialize the PCI library */ - pci_scan_bus(pacc); /* We want to get the list of devices */ - - pacc->error = pci_log; /* Replace logging and debug functions */ - pacc->warning = pci_log; - pacc->debug = pci_log; - pacc->debugging = 1; - - pci_scan_bus(pacc); /* We want to get the list of devices */ - - return pacc; -} - -void pci_release_handle() -{ - if (pacc) { - //pci_cleanup(pacc); - //pacc = NULL; - } -} - -struct pci_dev * pci_find_device(struct pci_access *pacc, struct pci_filter *f) -{ - struct pci_dev *d; - - /* Iterate over all devices */ - for (d = pacc->devices; d; d = d->next) { - if (pci_filter_match(f, d)) - return d; + dp = opendir(path); + if (dp == NULL) { + serror("Failed to detect PCI devices"); + return -1; } - return NULL; + while ((entry = readdir(dp))) { + struct pci_dev d; + + struct { const char *s; int *p; } map[] = { + { "vendor", &d.id.vendor }, + { "device", &d.id.device } + }; + + /* Read vendor & device id */ + for (int i = 0; i < 2; i++) { + snprintf(path, sizeof(path), "%s/bus/pci/devices/%s/%s", SYSFS_PATH, entry->d_name, map[i].s); + + f = fopen(path, "r"); + if (!f) + serror("Failed to open '%s'", path); + + ret = fscanf(f, "%x", map[i].p); + if (ret != 1) + error("Failed to parse %s ID from: %s", map[i].s, path); + + fclose(f); + } + + /* Get slot id */ + ret = sscanf(entry->d_name, "%4x:%2x:%2x.%u", &d.slot.domain, &d.slot.bus, &d.slot.device, &d.slot.function); + if (ret != 4) + error("Failed to parse PCI slot number: %s", entry->d_name); + } + + closedir(dp); + + return 0; +} + +void pci_destroy(struct pci *p) +{ + list_destroy(&p->devices, NULL, true); +} + +int pci_dev_init(struct pci_dev *d) +{ + return 0; +} + +void pci_dev_destroy(struct pci_dev *d) +{ + +} + +int pci_dev_parse_slot(struct pci_dev *f, const char *s, const char **error) +{ + char *str = strdup(s); + char *colon = strrchr(str, ':'); + char *dot = strchr((colon ? colon + 1 : str), '.'); + char *mid = str; + char *e, *bus, *colon2; + + if (colon) { + *colon++ = 0; + mid = colon; + + colon2 = strchr(str, ':'); + if (colon2) { + *colon2++ = 0; + bus = colon2; + + if (str[0] && strcmp(str, "*")) { + long int x = strtol(str, &e, 16); + if ((e && *e) || (x < 0 || x > 0x7fffffff)) { + *error = "Invalid domain number"; + goto fail; + } + + f->slot.domain = x; + } + } + else + bus = str; + + if (bus[0] && strcmp(bus, "*")) { + long int x = strtol(bus, &e, 16); + if ((e && *e) || (x < 0 || x > 0xff)) { + *error = "Invalid bus number"; + goto fail; + } + + f->slot.bus = x; + } + } + + if (dot) + *dot++ = 0; + + if (mid[0] && strcmp(mid, "*")) { + long int x = strtol(mid, &e, 16); + + if ((e && *e) || (x < 0 || x > 0x1f)) { + *error = "Invalid slot number"; + goto fail; + } + + f->slot.device = x; + } + + if (dot && dot[0] && strcmp(dot, "*")) { + long int x = strtol(dot, &e, 16); + + if ((e && *e) || (x < 0 || x > 7)) { + *error = "Invalid function number"; + goto fail; + } + + f->slot.function = x; + } + + free(str); + return 0; + +fail: + free(str); + return -1; +} + +/* ID filter syntax: [vendor]:[device][:class] */ +int pci_dev_parse_id(struct pci_dev *f, const char *str, const char **error) +{ + char *s, *c, *e; + + if (!*str) + return 0; + + s = strchr(str, ':'); + if (!s) { + *error = "':' expected"; + goto fail; + } + + *s++ = 0; + if (str[0] && strcmp(str, "*")) { + long int x = strtol(str, &e, 16); + + if ((e && *e) || (x < 0 || x > 0xffff)) { + *error = "Invalid vendor ID"; + goto fail; + } + + f->id.vendor = x; + } + + c = strchr(s, ':'); + if (c) + *c++ = 0; + + if (s[0] && strcmp(s, "*")) { + long int x = strtol(s, &e, 16); + if ((e && *e) || (x < 0 || x > 0xffff)) { + *error = "Invalid device ID"; + goto fail; + } + + f->id.device = x; + } + + if (c && c[0] && strcmp(s, "*")) { + long int x = strtol(c, &e, 16); + + if ((e && *e) || (x < 0 || x > 0xffff)) { + *error = "Invalid class code"; + goto fail; + } + + f->id.class = x; + } + + return 0; + +fail: + return -1; +} + +int pci_dev_compare(const struct pci_dev *d, const struct pci_dev *f) +{ + if ((f->slot.domain >= 0 && f->slot.domain != d->slot.domain) || + (f->slot.bus >= 0 && f->slot.bus != d->slot.bus) || + (f->slot.device >= 0 && f->slot.device != d->slot.device) || + (f->slot.function >= 0 && f->slot.function != d->slot.function)) + return 0; + + if (f->id.device >= 0 || f->id.vendor >= 0) { + if ((f->id.device >= 0 && f->id.device != d->id.device) || (f->id.vendor >= 0 && f->id.vendor != d->id.vendor)) + return 0; + } + + if (f->id.class >= 0) { + if (f->id.class != d->id.class) + return 0; + } + + return 1; +} + +struct pci_dev * pci_lookup_device(struct pci *p, struct pci_dev *f) +{ + return list_search(&p->devices, (cmp_cb_t) pci_dev_compare, (void *) f); } int pci_attach_driver(struct pci_dev *d, const char *driver) @@ -84,8 +249,8 @@ int pci_attach_driver(struct pci_dev *d, const char *driver) if (!f) serror("Failed to add PCI id to %s driver (%s)", driver, fn); - debug(5, "Adding ID to %s module: %04x %04x", driver, d->vendor_id, d->device_id); - fprintf(f, "%04x %04x", d->vendor_id, d->device_id); + debug(5, "Adding ID to %s module: %04x %04x", driver, d->id.vendor, d->id.device); + fprintf(f, "%04x %04x", d->id.vendor, d->id.device); fclose(f); /* Bind to driver */ @@ -95,19 +260,19 @@ int pci_attach_driver(struct pci_dev *d, const char *driver) serror("Failed to bind PCI device to %s driver (%s)", driver, fn); debug(5, "Bind device to %s driver", driver); - fprintf(f, "%04x:%02x:%02x.%x\n", d->domain, d->bus, d->dev, d->func); + fprintf(f, "%04x:%02x:%02x.%x\n", d->slot.domain, d->slot.bus, d->slot.device, d->slot.function); fclose(f); return 0; } -int pci_get_iommu_group(struct pci_dev *pdev) +int pci_get_iommu_group(struct pci_dev *d) { int ret; char *group, link[1024], sysfs[1024]; snprintf(sysfs, sizeof(sysfs), "%s/bus/pci/devices/%04x:%02x:%02x.%x/iommu_group", SYSFS_PATH, - pdev->domain, pdev->bus, pdev->dev, pdev->func); + d->slot.domain, d->slot.bus, d->slot.device, d->slot.function); ret = readlink(sysfs, link, sizeof(link)); if (ret < 0) diff --git a/lib/kernel/vfio.c b/lib/kernel/vfio.c index aee6d74f3..68e7e0b3d 100644 --- a/lib/kernel/vfio.c +++ b/lib/kernel/vfio.c @@ -225,7 +225,7 @@ int vfio_pci_attach(struct vfio_dev *d, struct vfio_container *c, struct pci_dev error("Failed to get IOMMU group of device"); /* VFIO device name consists of PCI BDF */ - snprintf(name, sizeof(name), "%04x:%02x:%02x.%x", pdev->domain, pdev->bus, pdev->dev, pdev->func); + snprintf(name, sizeof(name), "%04x:%02x:%02x.%x", pdev->slot.domain, pdev->slot.bus, pdev->slot.device, pdev->slot.function); ret = vfio_dev_attach(d, c, name, index); if (ret < 0) diff --git a/lib/nodes/fpga.c b/lib/nodes/fpga.c index 9c3851db0..def80ce47 100644 --- a/lib/nodes/fpga.c +++ b/lib/nodes/fpga.c @@ -22,6 +22,7 @@ #include "timing.h" struct fpga fpga; +struct pci pci; struct vfio_container vc; int fpga_reset(struct fpga *f) @@ -55,21 +56,12 @@ int fpga_reset(struct fpga *f) void fpga_dump(struct fpga *f) { - char namebuf[128]; - char *name; - - struct pci_access *pacc; - - pacc = pci_get_handle(); - name = pci_lookup_name(pacc, namebuf, sizeof(namebuf), PCI_LOOKUP_DEVICE, fpga.vd.pdev->vendor_id, fpga.vd.pdev->device_id); - pci_fill_info(fpga.vd.pdev, PCI_FILL_IDENT | PCI_FILL_BASES | PCI_FILL_CLASS); /* Fill in header info we need */ - - info("VILLASfpga card: %s", name); + info("VILLASfpga card:"); { INDENT - info("Slot: %04x:%02x:%02x.%d", fpga.vd.pdev->domain, fpga.vd.pdev->bus, fpga.vd.pdev->dev, fpga.vd.pdev->func); - info("Vendor ID: %04x", fpga.vd.pdev->vendor_id); - info("Device ID: %04x", fpga.vd.pdev->device_id); - info("Class ID: %04x", fpga.vd.pdev->device_class); + info("Slot: %04x:%02x:%02x.%d", fpga.vd.pdev->slot.domain, fpga.vd.pdev->slot.bus, fpga.vd.pdev->slot.device, fpga.vd.pdev->slot.function); + info("Vendor ID: %04x", fpga.vd.pdev->id.vendor); + info("Device ID: %04x", fpga.vd.pdev->id.device); + info("Class ID: %04x", fpga.vd.pdev->id.class); info("BAR0 mapped at %p", fpga.map); @@ -94,8 +86,8 @@ int fpga_parse_card(struct fpga *f, int argc, char * argv[], config_setting_t *c config_setting_t *cfg_ips, *cfg_slot, *cfg_id, *cfg_fpgas; /* Default values */ - f->filter.vendor = FPGA_PCI_VID_XILINX; - f->filter.device = FPGA_PCI_PID_VFPGA; + f->filter.id.vendor = FPGA_PCI_VID_XILINX; + f->filter.id.device = FPGA_PCI_PID_VFPGA; cfg_fpgas = config_setting_get_member(cfg, "fpgas"); if (!cfg_fpgas) @@ -118,24 +110,24 @@ int fpga_parse_card(struct fpga *f, int argc, char * argv[], config_setting_t *c if (cfg_slot) { slot = config_setting_get_string(cfg_slot); if (slot) { - err = pci_filter_parse_slot(&f->filter, (char*) slot); - if (err) - cerror(cfg_slot, "%s", err); + ret = pci_dev_parse_slot(&f->filter, slot, &err); + if (ret) + cerror(cfg_slot, "Failed to parse PCI slot: %s", err); } else - cerror(cfg_slot, "Invalid slot format"); + cerror(cfg_slot, "PCI slot must be a string"); } cfg_id = config_setting_get_member(f->cfg, "id"); if (cfg_id) { id = config_setting_get_string(cfg_id); if (id) { - err = pci_filter_parse_id(&f->filter, (char*) id); - if (err) - cerror(cfg_id, "%s", err); + ret = pci_dev_parse_id(&f->filter, (char*) id, &err); + if (ret) + cerror(cfg_id, "Failed to parse PCI id: %s", err); } else - cerror(cfg_slot, "Invalid id format"); + cerror(cfg_slot, "PCI ID must be a string"); } cfg_ips = config_setting_get_member(f->cfg, "ips"); @@ -162,16 +154,15 @@ int fpga_parse_card(struct fpga *f, int argc, char * argv[], config_setting_t *c int fpga_init(int argc, char * argv[], config_setting_t *cfg) { int ret; - struct pci_access *pacc; - struct pci_dev *pdev; struct fpga *f; + struct pci_dev *pdev; /* For now we only support a single VILALSfpga card */ f = fpga_get(); list_init(&f->ips); - pacc = pci_get_handle(); - pci_filter_init(pacc, &f->filter); + pci_init(&pci); + pci_dev_init(&f->filter); /* Parse FPGA configuration */ ret = fpga_parse_card(f, argc, argv, cfg); @@ -192,7 +183,7 @@ int fpga_init(int argc, char * argv[], config_setting_t *cfg) warn("FPGA is missing an AXI4-Stream switch"); /* Search for FPGA card */ - pdev = pci_find_device(pacc, &f->filter); + pdev = pci_lookup_device(&pci, &f->filter); if (!pdev) error("Failed to find PCI device"); @@ -243,8 +234,8 @@ int fpga_deinit() int ret; list_destroy(&fpga.ips, (dtor_cb_t) ip_destroy, true); - - pci_release_handle(); + + pci_destroy(&pci); ret = vfio_destroy(&vc); if (ret) @@ -276,9 +267,10 @@ char * fpga_print(struct node *n) if (d->ip) return strf("dm=%s (%s:%s:%s:%s) baseaddr=%#jx port=%u slot=%02"PRIx8":%02"PRIx8".%"PRIx8" id=%04"PRIx16":%04"PRIx16, - d->ip->name, d->ip->vlnv.vendor, d->ip->vlnv.library, d->ip->vlnv.name, d->ip->vlnv.version, d->ip->baseaddr, d->ip->port, - f->filter.bus, f->filter.device, f->filter.func, - f->filter.vendor, f->filter.device); + d->ip->name, d->ip->vlnv.vendor, d->ip->vlnv.library, d->ip->vlnv.name, d->ip->vlnv.version, + d->ip->baseaddr, d->ip->port, + f->filter.slot.bus, f->filter.slot.device, f->filter.slot.function, + f->filter.id.vendor, f->filter.id.device); else return strf("dm=%s", d->ip_name); } diff --git a/thirdparty/Makefile b/thirdparty/Makefile index 06b445bd9..6150cdf09 100644 --- a/thirdparty/Makefile +++ b/thirdparty/Makefile @@ -1,4 +1,4 @@ -DEPS = libxil libconfig libnl libwebsockets pciutils criterion +DEPS = libxil libconfig libnl libwebsockets criterion .PHONY: $(DEPS) all clean @@ -23,14 +23,6 @@ libnl: ./configure --prefix=$(PREFIX) --disable-cli && \ make install -# Install & compile libpci dependency -pciutils: - cd $@ && \ - make clean && \ - make SHARED=yes && \ - make install-lib PREFIX=$(PREFIX) && \ - ln -s $(PREFIX)/lib/libpci.so.* $(PREFIX)/lib/libpci.so - # Install & compile libwebsockets dependency libwebsockets: mkdir $@/build && cd $@/build && \