diff --git a/server/etc/example.conf b/server/etc/example.conf index 30738dfe2..75790005d 100644 --- a/server/etc/example.conf +++ b/server/etc/example.conf @@ -77,7 +77,11 @@ nodes = { out = "" }, gtfpga_node = { + type = "gtfpga", + slot = "01:00.0", # The PCIe slot location (see first column in 'lspci' output) + id = "1ab8:4005", # The PCIe vendor:device ID (see third column in 'lspci -n' output) + rate = 1 } }; diff --git a/server/include/gtfpga.h b/server/include/gtfpga.h index 7b3f60ea9..593f7170e 100644 --- a/server/include/gtfpga.h +++ b/server/include/gtfpga.h @@ -1,6 +1,9 @@ /** Node type: GTFPGA (Xilinx ML507) * * This file implements the gtfpga subtype for nodes. + * It's based on the uio_pci_generic kernel module. + * A detailed description of that module is available here: + * http://www.hep.by/gnu/kernel/uio-howto/uio_pci_generic.html * * @author Steffen Vogel * @copyright 2015, Institute for Automation of Complex Power Systems, EONERC @@ -14,24 +17,34 @@ #include -#define GTFPGA_BAR 0 /**< The Base Address Register which is mmap()ed to the User Space */ +#include "node.h" -#define GTFPGA_MAX_TX 64 /**< The amount of values which is supported by the GTFPGA card */ -#define GTFPGA_MAX_RX 64 /**< The amount of values which is supported by the GTFPGA card */ +#define GTFPGA_BAR 0 /**< The Base Address Register which is mmap()ed to the User Space */ -#define GTFPGA_VID 0x10ee /**< The default vendor ID of the GTFPGA card */ -#define GTFPGA_DID 0x0007 /**< The default device ID of the GTFPGA card */ +#define GTFPGA_MAX_TX 64 /**< The amount of values which is supported by the GTFPGA card */ +#define GTFPGA_MAX_RX 64 /**< The amount of values which is supported by the GTFPGA card */ +#define GTFPGA_VID 0x10ee /**< The default vendor ID of the GTFPGA card */ +#define GTFPGA_DID 0x0007 /**< The default device ID of the GTFPGA card */ struct gtfpga { struct pci_filter filter; - int fd_mmap, fd_uio; + int fd_mmap; /**< File descriptor for the memory mapped PCI BAR */ void *map; + + /* The following descriptor is blocking as long no interrupt was received + * or the timer has not elapsed */ + int fd_irq; /**< File descriptor for the timer */ + + char *name; + double rate; struct pci_dev *dev; }; +typedef void(*log_cb_t)(char *, ...); + /** @see node_vtable::init */ int gtfpga_init(int argc, char * argv[], struct settings *set); @@ -51,9 +64,9 @@ int gtfpga_open(struct node *n); int gtfpga_close(struct node *n); /** @see node_vtable::read */ -int gtfpga_read(struct node *n, struct msg *m); +int gtfpga_read(struct node *n, struct msg *pool, int poolsize, int first, int cnt); /** @see node_vtable::write */ -int gtfpga_write(struct node *n, struct msg *m); +int gtfpga_write(struct node *n, struct msg *pool, int poolsize, int first, int cnt); #endif /** _GTFPGA_H_ @} */ diff --git a/server/src/gtfpga.c b/server/src/gtfpga.c index bd579756e..af6983e0d 100644 --- a/server/src/gtfpga.c +++ b/server/src/gtfpga.c @@ -7,34 +7,54 @@ */ #include "gtfpga.h" +#include "utils.h" + +#include +#include +#include +#include +#include #define SYSFS_PATH "/sys/bus/pci" -static pci_access *pacc; +static struct pci_access *pacc; -int gtfpga_init(int argc, char *argv[]) +static void gtfpga_debug(char *msg, ...) { + va_list ap; + + va_start(ap, msg); + log_vprint(DEBUG, msg, ap); + va_end(ap); +} + +int gtfpga_init(int argc, char * argv[], struct settings *set) { pacc = pci_alloc(); /* Get the pci_access structure */ pci_init(pacc); /* Initialize the PCI library */ - pacc->error = error; /* Replace logging and debug functions */ - pacc->warning = warn; - pacc->debug = debug; + pacc->error = (log_cb_t) error; /* Replace logging and debug functions */ + pacc->warning = (log_cb_t) warn; + pacc->debug = gtfpga_debug; pci_scan_bus(pacc); /* We want to get the list of devices */ + + return 0; } int gtfpga_deinit() { pci_cleanup(pacc); + + return 0; } int gtfpga_parse(config_setting_t *cfg, struct node *n) { - char *slot, *id; - config_setting_t *cfg_slot, *cfg_id; struct gtfpga *g = alloc(sizeof(struct gtfpga)); + const char *slot, *id, *err; + config_setting_t *cfg_slot, *cfg_id; + /* Checks */ if (n->combine != 1) { config_setting_t *cfg_combine = config_setting_get_member(cfg, "combine"); @@ -43,31 +63,53 @@ int gtfpga_parse(config_setting_t *cfg, struct node *n) pci_filter_init(NULL, &g->filter); - if (cfg_slot = config_setting_get_member(cfg, "slot")) { - if (slot = config_setting_get_string(cfg_slot)) { - if ((err = pci_filter_parse_slot(&g->filter, slot)) + cfg_slot = config_setting_get_member(cfg, "slot"); + if (cfg_slot) { + slot = config_setting_get_string(cfg_slot); + if (slot) { + err = pci_filter_parse_slot(&g->filter, (char*) slot); + if (err) cerror(cfg_slot, "%s", err); } else cerror(cfg_slot, "Invalid slot format"); } - if (cfg_id = config_setting_get_member(cfg, "id")) { - if (id = config_setting_get_string(cfg_id)) { - if ((err = pci_filter_parse_id(&g->filter, id)) + cfg_id = config_setting_get_member(cfg, "id"); + if (cfg_id) { + id = config_setting_get_string(cfg_id); + if (id) { + err = pci_filter_parse_id(&g->filter, (char*) id); + if (err) cerror(cfg_id, "%s", err); } else cerror(cfg_slot, "Invalid id format"); } - + + if (!config_setting_lookup_float(cfg, "rate", &g->rate)) + g->rate = 0; + + n->gtfpga = g; return 0; } int gtfpga_print(struct node *n, char *buf, int len) { - + struct gtfpga *g = n->gtfpga; + + if (g->dev) { + return snprintf(buf, len, "rate=%.1f slot=%04"PRIx16":%02"PRIx8":%02"PRIx8".%"PRIx8 + " id=%04"PRIx16":%04"PRIx16" class=%04"PRIx16" irq=%d (%s)", g->rate, + g->dev->domain, g->dev->bus, g->dev->dev, g->dev->func, g->dev->vendor_id, g->dev->device_id, + g->dev->device_class, g->dev->irq, g->name); + } + else { + return snprintf(buf, len, "rate=%.1f slot=%02"PRIx8":%02"PRIx8".%"PRIx8" id=%04"PRIx16":%04"PRIx16, g->rate, + g->filter.bus, g->filter.device, g->filter.func, + g->filter.vendor, g->filter.device); + } } static int gtfpga_load_driver(struct pci_dev *d) @@ -78,7 +120,7 @@ static int gtfpga_load_driver(struct pci_dev *d) /* Prepare slot identifier */ snprintf(slot, sizeof(slot), "%04x:%02x:%02x.%x", - d->domain, d->bus, d->slot, d->func); + d->domain, d->bus, d->dev, d->func); /* Load uio_pci_generic module */ ret = system2("modprobe uio_pci_generic"); @@ -90,6 +132,7 @@ static int gtfpga_load_driver(struct pci_dev *d) if (!f) serror("Failed to add PCI id to uio_pci_generic driver"); + debug(5, "Adding ID to uio_pci_generic module: %04x %04x", d->vendor_id, d->device_id); fprintf(f, "%04x %04x", d->vendor_id, d->device_id); fclose(f); @@ -98,8 +141,11 @@ static int gtfpga_load_driver(struct pci_dev *d) if (!f) serror("Failed to add PCI id to uio_pci_generic driver"); + debug(5, "Bind slot to uio_pci_generic module: %s", slot); fprintf(f, "%s\n", slot); fclose(f); + + return 0; } static struct pci_dev * gtfpga_find_device(struct pci_filter *f) @@ -108,17 +154,15 @@ static struct pci_dev * gtfpga_find_device(struct pci_filter *f) /* Iterate over all devices */ for (d = pacc->devices; d; d = d->next) { - if (pci_filter_match(&f, d)) + if (pci_filter_match(f, d)) return d; } return NULL; } -static int gtfpga_mmap(struct node *n) +static int gtfpga_mmap(struct gtfpga *g) { - struct gtfpga *g = n->gtfpga; - int fd = open("/dev/mem", O_RDWR | O_SYNC); if (!fd) serror("Failed open()"); @@ -127,10 +171,12 @@ static int gtfpga_mmap(struct node *n) int size = g->dev->size[GTFPGA_BAR]; /* mmap() first BAR */ - printf("mmap(NULL, %#x, PROT_READ | PROT_WRITE, MAP_SHARED, %u, %#lx)", size, fd, addr); + debug(5, "Setup mapping: mmap(NULL, %#x, PROT_READ | PROT_WRITE, MAP_SHARED, %u, %#lx)", size, fd, addr); void *map = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, addr); if (map == MAP_FAILED) serror("Failed mmap()"); + + return 0; } int gtfpga_open(struct node *n) @@ -140,29 +186,43 @@ int gtfpga_open(struct node *n) int ret; - dev = gtfpga_find_device(g->filter); + dev = gtfpga_find_device(&g->filter); if (!dev) - error("No GTFPGA card detected"); + error("No GTFPGA card found"); g->dev = dev; + g->name = alloc(512); - ret = gtfpga_load_driver(dev); - if (ret) - error("Failed to load and bind driver (uio_pci_generic)"); - - ret = gtfpga_mmap(g); - if (ret) - error("Failed to setup memory mapping for GTFGPA card"); + gtfpga_load_driver(dev); + gtfpga_mmap(g); /* Show some debug infos */ pci_fill_info(dev, PCI_FILL_IDENT | PCI_FILL_BASES | PCI_FILL_CLASS); /* Fill in header info we need */ - - debug(3, "Found GTFPGA card: %04x:%02x:%02x.%d vendor=%04x device=%04x class=%04x irq=%d", - dev->domain, dev->bus, dev->dev, dev->func, dev->vendor_id, dev->device_id, - dev->device_class, dev->irq); - - debug(3, " (%s)\n", pci_lookup_name(pacc, namebuf, sizeof(namebuf), PCI_LOOKUP_DEVICE, dev->vendor_id, dev->device_id)); - + + g->name = pci_lookup_name(pacc, g->name, 512, PCI_LOOKUP_DEVICE, dev->vendor_id, dev->device_id); + + /* Setup timer */ + if (g->rate) { + g->fd_irq = timerfd_create(CLOCK_MONOTONIC, 0); + if (g->fd_irq < 0) + serror("Failed to create timer"); + + struct itimerspec its = { + .it_interval = timespec_rate(g->rate), + .it_value = { 1, 0 } + }; + ret = timerfd_settime(g->fd_irq, 0, &its, NULL); + if (ret) + serror("Failed to start timer"); + } + else + /** @todo implement UIO interrupts */ + error("UIO irq not implemented yet. Use 'rate' setting"); + + char buf[1024]; + gtfpga_print(n, buf, sizeof(buf)); + debug(5, "Found GTFPGA card: %s", buf); + return 0; } @@ -174,23 +234,35 @@ int gtfpga_close(struct node *n) munmap(g->map, g->dev->size[GTFPGA_BAR]); close(g->fd_mmap); - close(g->fd_uio); + close(g->fd_irq); + free(g->name); return 0; } /** @todo implement */ -int gtfpga_read(struct node *n, struct msg *m) +int gtfpga_read(struct node *n, struct msg *pool, int poolsize, int first, int cnt) { struct gtfpga *g = n->gtfpga; + + struct msg *m = &pool[first % poolsize]; + + uint64_t runs; + read(g->fd_irq, &runs, sizeof(runs)); + + static int seq; + + m->sequence = seq++; return 0; } /** @todo implement */ -int gtfpga_write(struct node *n, struct msg *m) +int gtfpga_write(struct node *n, struct msg *pool, int poolsize, int first, int cnt) { - struct gtfpga *g = n->gtfpga; + // struct gtfpga *g = n->gtfpga; + + // struct msg *m = &pool[first % poolsize]; return 0; }