/** Node type: VILLASfpga * * @author Steffen Vogel * @copyright 2015-2016, Steffen Vogel * This file is part of VILLASnode. All Rights Reserved. Proprietary and confidential. * Unauthorized copying of this file, via any medium is strictly prohibited. *********************************************************************************/ #include #include #include #include #include #include "kernel/kernel.h" #include "kernel/pci.h" #include "nodes/vfpga.h" #include "fpga/ip.h" #include "config-fpga.h" #include "utils.h" #include "timing.h" struct vfpga fpga; struct vfio_container vc; static struct pci_access *pacc; typedef void(*log_cb_t)(char *, ...); int vfpga_reset(struct vfpga *f) { int ret; char state[4096]; /* Save current state of PCI configuration space */ ret = pread(f->vd.fd, state, sizeof(state), (off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40); if (ret != sizeof(state)) return -1; uint32_t *rst_reg = (uint32_t *) (f->map + f->baseaddr.reset); info("Reset fpga"); rst_reg[0] = 1; usleep(100000); /* Restore previous state of PCI configuration space */ ret = pwrite(f->vd.fd, state, sizeof(state), (off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40); if (ret != sizeof(state)) return -1; info("Reset status = %#x", *rst_reg); return 0; } void vfpga_dump(struct vfpga *f) { char namebuf[128]; char *name; name = pci_lookup_name(pacc, namebuf, sizeof(namebuf), PCI_LOOKUP_DEVICE, fpga.vd.pdev->vendor_id, fpga.vd.pdev->device_id); pci_fill_info(fpga.vd.pdev, PCI_FILL_IDENT | PCI_FILL_BASES | PCI_FILL_CLASS); /* Fill in header info we need */ info("VILLASfpga card: %s", name); { INDENT info("Slot: %04x:%02x:%02x.%d", fpga.vd.pdev->domain, fpga.vd.pdev->bus, fpga.vd.pdev->dev, fpga.vd.pdev->func); info("Vendor ID: %04x", fpga.vd.pdev->vendor_id); info("Device ID: %04x", fpga.vd.pdev->device_id); info("Class ID: %04x", fpga.vd.pdev->device_class); info("BAR0 mapped at %p", fpga.map); info("DMA mapped at %p", fpga.dma); info("IP blocks:"); list_foreach(struct ip *i, &f->ips) { INDENT ip_dump(i); } } vfio_dump(fpga.vd.group->container); } struct vfpga * vfpga_get() { return &fpga; } static void vfpga_debug(char *msg, ...) { va_list ap; va_start(ap, msg); log_vprint(LOG_LVL_DEBUG, msg, ap); va_end(ap); } int vfpga_parse(struct vfpga *v, int argc, char * argv[], config_setting_t *cfg) { int ret; const char *slot, *id, *err; config_setting_t *cfg_fpga, *cfg_ips, *cfg_slot, *cfg_id; /* Default values */ v->filter.vendor = PCI_VID_XILINX; v->filter.device = PCI_PID_VFPGA; v->reset = false; cfg_fpga = config_setting_get_member(cfg, "fpga"); if (!cfg_fpga) cerror(cfg, "Config file is missing VILLASfpga configuration"); config_setting_lookup_bool(cfg_fpga, "reset", &v->reset); cfg_slot = config_setting_get_member(cfg_fpga, "slot"); if (cfg_slot) { slot = config_setting_get_string(cfg_slot); if (slot) { err = pci_filter_parse_slot(&v->filter, (char*) slot); if (err) cerror(cfg_slot, "%s", err); } else cerror(cfg_slot, "Invalid slot format"); } cfg_id = config_setting_get_member(cfg_fpga, "id"); if (cfg_id) { id = config_setting_get_string(cfg_id); if (id) { err = pci_filter_parse_id(&v->filter, (char*) id); if (err) cerror(cfg_id, "%s", err); } else cerror(cfg_slot, "Invalid id format"); } cfg_ips = config_setting_get_member(cfg_fpga, "ips"); if (!cfg_ips) cerror(cfg_fpga, "FPGA configuration is missing ips section"); for (int i = 0; i < config_setting_length(cfg_ips); i++) { struct ip ip; config_setting_t *cfg_ip = config_setting_get_elem(cfg_ips, i); ret = ip_parse(&ip, cfg_ip); if (ret) cerror(cfg_ip, "Failed to parse VILLASfpga ip"); list_push(&v->ips, memdup(&ip, sizeof(struct ip))); } return 0; } struct pci_access * vfpga_init_pci() { struct pci_access *pacc; pacc = pci_alloc(); /* Get the pci_access structure */ if (!pacc) error("Failed to allocate PCI access structure"); pci_init(pacc); /* Initialize the PCI library */ pci_scan_bus(pacc); /* We want to get the list of devices */ pacc->error = (log_cb_t) error; /* Replace logging and debug functions */ pacc->warning = (log_cb_t) warn; pacc->debug = vfpga_debug; pci_scan_bus(pacc); /* We want to get the list of devices */ return pacc; } int vfpga_init(int argc, char * argv[], config_setting_t *cfg) { int ret; struct pci_access *pacc; struct pci_dev *pdev; struct vfpga *f; /* For now we only support a single VILALSfpga card */ f = vfpga_get(); /* Some hardcoded addresses for internal IP blocks */ f->baseaddr.reset = 0x2000; f->baseaddr.intc = 0x5000; pacc = vfpga_init_pci(); pci_filter_init(pacc, &f->filter); list_init(&f->ips); ret = vfpga_parse(f, argc, argv, cfg); if (ret) cerror(cfg, "Failed to parse VILLASfpga config"); /* Search for fpga card */ pdev = pci_find_device(pacc, &f->filter); if (!pdev) error("Failed to find PCI device"); if (pdev->vendor_id != PCI_VID_XILINX) error("This is not a Xilinx FPGA board!"); /* Get VFIO handles and details */ ret = vfio_init(&vc); if (ret) serror("Failed to initialize VFIO"); /* Attach PCIe card to VFIO container */ ret = vfio_pci_attach(&f->vd, &vc, pdev); if (ret) error("Failed to attach VFIO device"); /* Setup IRQs */ for (int i = 0; i < f->vd.irqs[VFIO_PCI_MSI_IRQ_INDEX].count; i++) { ret = vfio_pci_msi_fd(&f->vd, (1 << i)); if (ret < 0) serror("Failed to create eventfd for IRQ: ret=%d", f->vd.msi_efds[i]); } /* Map PCIe BAR */ f->map = vfio_map_region(&f->vd, VFIO_PCI_BAR0_REGION_INDEX); if (f->map == MAP_FAILED) serror("Failed to mmap() BAR0"); /* Map DMA accessible memory */ f->dma = vfio_map_dma(f->vd.group->container, 0x10000, 0x1000, 0x0); if (f->dma == MAP_FAILED) serror("Failed to mmap() DMA"); /* Enable memory access and PCI bus mastering for DMA */ ret = vfio_pci_enable(&f->vd); if (ret) serror("Failed to enable PCI device"); /* Trigger internal reset of fpga card / PCIe endpoint */ if (f->reset) { ret = vfpga_reset(f); if (ret) serror("Failed to reset fpga card"); /* Reset / detect PCI device */ ret = vfio_pci_reset(&f->vd); if (ret) serror("Failed to reset PCI device"); } list_foreach(struct ip *c, &f->ips) { c->card = f; ip_init(c); } return 0; } int vfpga_deinit() { int ret; list_destroy(&fpga.ips, (dtor_cb_t) ip_destroy, true); pci_cleanup(pacc); ret = vfio_destroy(&vc); if (ret) error("Failed to deinitialize VFIO module"); return 0; } int vfpga_parse_node(struct node *n, config_setting_t *cfg) { // struct vfpga *v = n->_vd; return 0; } char * vfpga_print(struct node *n) { struct vfpga *v = n->_vd; return strf("slot=%02"PRIx8":%02"PRIx8".%"PRIx8" id=%04"PRIx16":%04"PRIx16, v->filter.bus, v->filter.device, v->filter.func, v->filter.vendor, v->filter.device); } int vfpga_open(struct node *n) { struct vfpga *v = n->_vd; struct pci_dev *dev; dev = pci_find_device(pacc, &v->filter); if (!dev) error("No vfpga card found"); /* @todo */ return 0; } int vfpga_close(struct node *n) { // struct vfpga *v = n->_vd; /* @todo */ return 0; } int vfpga_read(struct node *n, struct sample *smps[], unsigned cnt) { // struct vfpga *v = n->_vd; /** @todo */ return 0; } /** @todo implement */ int vfpga_write(struct node *n, struct sample *smps[], unsigned cnt) { // struct vfpga *v = n->_vd; /** @todo */ return 0; } static struct node_type vt = { .name = "vfpga", .description = "VILLASfpga PCIe card (libpci)", .vectorize = 1, .parse = vfpga_parse_node, .print = vfpga_print, .open = vfpga_open, .close = vfpga_close, .read = vfpga_read, .write = vfpga_write, .init = vfpga_init, .deinit = vfpga_deinit }; REGISTER_NODE_TYPE(&vt)