diff --git a/Makefile b/Makefile index f752a093b..911b1e821 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,8 @@ endif # Enable VILLASfpga support when libpci is available ifeq ($(shell pkg-config libpci; echo $$?),0) - LIB_OBJS += vfpga.o pci.o dma.o model.o fifo.o xsg.o vfio.o switch.o rtds_axis.o + LIB_OBJS += vfpga.o pci.o ip.o vfio.o + LIB_OBJS += dma.o model.o fifo.o switch.o rtds_axis.o LDLIBS += -lxil LDFLAGS += -Lthirdparty/xilinx -Wl,-rpath-link,'$$ORIGIN/thirdparty/xilinx' CFLAGS += -Ithirdparty/xilinx/include diff --git a/config-fpga.h b/config-fpga.h index 5644204c9..afc74b3e4 100644 --- a/config-fpga.h +++ b/config-fpga.h @@ -28,83 +28,6 @@ #define virt_to_dma(virt, dma) ((char *) ((char *) virt - dma) + BASEADDR_HOST) #define dma_to_virt(addr, dma) ((char *) ((char *) addr - BASEADDR_HOST) + dma) -#if 0 -/** Base addresses in AXI bus of FPGA */ -enum { - BASEADDR_BRAM = 0x0000, /* 8 KB Block RAM for DMA SG descriptors */ - BASEADDR_XSG = 0x2000, /* Xilinx System Generator Model */ - BASEADDR_DMA_SG = 0x3000, /* Scatter Gather DMA controller registers */ - BASEADDR_TIMER = 0x4000, /* Timer Counter registers */ - BASEADDR_SWITCH = 0x5000, /* AXI Stream switch registers */ - BASEADDR_FIFO_MM = 0x6000, /* Memory-mapped to AXI Stream FIFO */ - BASEADDR_RESET = 0x7000, /* Internal Reset register */ - BASEADDR_RTDS = 0x8000, /* RTDS to AXI Stream bridge */ - BASEADDR_HLS = 0x9000, /* High-level Synthesis model registers */ - BASEADDR_DMA_SIMPLE = 0xA000, /* Simple DMA controller registers */ - BASEADDR_INTC = 0xB000, /* PCIe MSI Interrupt controller registers */ - BASEADDR_FIFO_MM_AXI4 = 0xC000, /* AXI4 (full) interface of AXI Stream FIFO */ - BASEADDR_HOST = 0x80000000 /* Start of host memory (0x0 of IOMMU) */ -}; - -/** MSI vector table for VILLASfpga components */ -enum { - MSI_TMRCTR0 = 0, /* Sensivity: rising edge, synchronous: axi_clk */ - MSI_TMRCTR1 = 1, /* Sensivity: rising edge, synchronous: axi_clk */ - MSI_FIFO_MM = 2, /* Sensivity: level high, synchronous: axi_clk */ - MSI_DMA_MM2S = 3, /* Sensivity: level high, synchronous: axi_clk */ - MSI_DMA_S2MM = 4, /* Sensivity: level high, synchronous: axi_clk */ - MSI_RTDS_TS = 5, /* Sensivity: rising edge, synchronous: rtds_clk100m (New timestep started) */ - MSI_RTDS_OVF = 6, /* Sensivity: rising edge, synchronous: rtds_clk100m (UserTxInProgress overlapped with UserTStepPulse)*/ - MSI_RTDS_CASE = 7 /* Sensivity: rising edge, synchronous: rtds_clk100m (New RSCAD case started) */ -}; - -/** Mapping of VILLASfpga components to AXI Stream interconnect ports */ -enum { - AXIS_SWITCH_RTDS = 0, - AXIS_SWITCH_DMA_SG = 1, - AXIS_SWITCH_FIFO_MM = 2, - AXIS_SWITCH_FIFO0 = 3, - AXIS_SWITCH_XSG = 4, - AXIS_SWITCH_HLS = 5, - AXIS_SWITCH_FIFO1 = 6, - AXIS_SWITCH_DMA_SIMPLE = 7, - AXI_SWITCH_NUM = 8 /* Number of master and slave interfaces */ -}; -#else - #define HAS_DMA_SIMPLE 1 - #define HAS_SWITCH 1 - #define HAS_XSG 1 - -/** Base addresses in AXI bus of FPGA */ -enum { - BASEADDR_SWITCH = 0x0000, /* AXI Stream switch registers */ - BASEADDR_DMA_SIMPLE = 0x1000, /* Simple DMA controller registers */ - BASEADDR_RESET = 0x2000, /* Internal Reset register */ - BASEADDR_RTDS = 0x3000, /* RTDS to AXI Stream bridge */ - BASEADDR_XSG = 0x4000, /* Xilinx System Generator Model */ - BASEADDR_INTC = 0x5000, /* PCIe MSI Interrupt controller registers */ - BASEADDR_PCIE = 0x10000000, /* PCIe config space */ - BASEADDR_HOST = 0x80000000 /* Start of host memory (0x0 of IOMMU) */ -}; - -/** MSI vector table for VILLASfpga components */ -enum { - MSI_DMA_MM2S = 0, /* Sensivity: level high, synchronous: axi_clk */ - MSI_DMA_S2MM = 1, /* Sensivity: level high, synchronous: axi_clk */ - MSI_RTDS_TS = 2, /* Sensivity: rising edge, synchronous: rtds_clk100m (New timestep started) */ - MSI_RTDS_OVF = 3, /* Sensivity: rising edge, synchronous: rtds_clk100m (UserTxInProgress overlapped with UserTStepPulse)*/ - MSI_RTDS_CASE = 4 /* Sensivity: rising edge, synchronous: rtds_clk100m (New RSCAD case started) */ -}; - -/** Mapping of VILLASfpga components to AXI Stream interconnect ports */ -enum { - AXIS_SWITCH_RTDS = 0, - AXIS_SWITCH_XSG = 1, - AXIS_SWITCH_DMA_SIMPLE = 2, - AXI_SWITCH_NUM = 3 /* Number of master and slave interfaces */ -}; -#endif - #define PCI_VID_XILINX 0x10ee #define PCI_PID_VFPGA 0x7022 diff --git a/etc/vfpga.conf b/etc/vfpga.conf new file mode 100644 index 000000000..d97d1a393 --- /dev/null +++ b/etc/vfpga.conf @@ -0,0 +1,40 @@ +fpga = { + reset = false; + + ips = { + switch_0 = { + vlnv = "xilinx.com:ip:axis_interconnect:2.1" + baseaddr = 0x0000; + numports = 3; + }, + rtds_0 = { + vlnv = "acs.eonerc.rwth-aachen.de:user:rtds_axis:1.0" + baseaddr = 0x3000; + port = 0; + }, + dma_0 = { + vlnv = "xilinx.com:ip:axi_dma:7.1"; + baseaddr = 0x0000; + port = 2; + irq = 0 + }, + hls_dft_0 = { + vlnv = "acs.eonerc.rwth-aachen.de:hls:hls_dft:1.0"; + baseaddr = 0x0000; + port = 1; + switch = "switch_0"; + } + } + + /* Configure switch */ + paths = ( + { in = "dma", out = "hls_dft" }, + { in = "hls_dft", out = "dma" } + ) +} + +nodes = { + fpga_dma = { + + } +} \ No newline at end of file diff --git a/include/fpga-bench.h b/include/fpga-bench.h deleted file mode 100644 index a6a243a41..000000000 --- a/include/fpga-bench.h +++ /dev/null @@ -1,21 +0,0 @@ -/** Benchmarks for VILLASfpga - * - * @file - * @author Steffen Vogel - * @copyright 2015-2016, Steffen Vogel - * This file is part of VILLASnode. All Rights Reserved. Proprietary and confidential. - * Unauthorized copying of this file, via any medium is strictly prohibited. - **********************************************************************************/ - -#ifndef _BENCH_H_ -#define _BENCH_H_ - -#include "nodes/vfpga.h" - -int bench_memcpy(struct vfpga *f); - -int bench_memcpy(struct vfpga *f); - -int bench_dm(struct vfpga *f); - -#endif /* _BENCH_H_ */ \ No newline at end of file diff --git a/include/fpga-tests.h b/include/fpga-tests.h deleted file mode 100644 index c79eadc62..000000000 --- a/include/fpga-tests.h +++ /dev/null @@ -1,29 +0,0 @@ -/** Test procedures for VILLASfpga - * - * @file - * @author Steffen Vogel - * @copyright 2016, Steffen Vogel - * This file is part of VILLASnode. All Rights Reserved. Proprietary and confidential. - * Unauthorized copying of this file, via any medium is strictly prohibited. - *********************************************************************************/ - -#ifndef _TESTS_H_ -#define _TESTS_H_ - -#include "nodes/vfpga.h" - -int test_intc(struct vfpga *f); - -int test_xsg(struct vfpga *f, const char *name); - -int test_fifo(struct vfpga *f); - -int test_dma(struct vfpga *f); - -int test_timer(struct vfpga *f); - -int test_rtds_rtt(struct vfpga *f); - -int test_rtds_cbuilder(struct vfpga *f); - -#endif /* _TESTS_H_ */ \ No newline at end of file diff --git a/include/villas/fpga/model.h b/include/villas/fpga/model.h index c11e4a066..9c636977d 100644 --- a/include/villas/fpga/model.h +++ b/include/villas/fpga/model.h @@ -13,27 +13,19 @@ #include #include -#include +#include "list.h" -#include "xsg.h" +#define XSG_MAGIC 0xDEADBABE enum model_type { MODEL_TYPE_HLS, MODEL_TYPE_XSG }; -struct model { - char *name; /**< Name / Identifier of model */ - void *baseaddr; /**< Base of register map. */ - - enum model_type type; /**< Model type. */ - - struct list parameters; /**< List of model parameters. */ - - union { - struct xsg_model xsg; /**< XSG specific model data */ - //struct hls_model hls; /**< HLS specific model data */ - }; +enum model_xsg_block_type { + XSG_BLOCK_GATEWAY_IN = 0x1000, + XSG_BLOCK_GATEWAY_OUT = 0x1001, + XSG_BLOCK_INFO = 0x2000 }; enum model_param_type { @@ -56,6 +48,31 @@ union model_param_value { bool bol; }; +struct model { + char *xml; /**< Path to a XML file which describes this model */ + enum model_type type; + struct list parameters; /**< List of model parameters. */ + struct list infos; + + char *map; + size_t maplen; + + union { + struct xsg_model { + char *map; + size_t maplen; + } xsg; /**< XSG specific model data */ + struct hls_model { + + } hls; /**< HLS specific model data */ + }; +}; + +struct model_info { + char *field; + char *value; +}; + struct model_param { char *name; /**< Name of the parameter */ @@ -63,25 +80,25 @@ struct model_param { enum model_param_type type; /**< Data type. Integers are represented by MODEL_GW_TYPE_(U)FIX with model_gw::binpt == 0 */ int binpt; /**< Binary point for type == MODEL_GW_TYPE_(U)FIX */ - int offset; /**< Register offset to model::baseaddress */ + uintptr_t offset; /**< Register offset to model::baseaddress */ union model_param_value default_value; struct model *model; /**< A pointer to the model structure to which this parameters belongs to. */ }; -int model_init(struct model *m, const char *xml); +int model_init(struct model *m, uintptr_t baseaddr); + +int model_init_from_xsg_map(struct model *m, uintptr_t baseaddr); + +int model_init_from_xml(struct model *m); void model_destroy(struct model *m); void model_dump(struct model *m); -void model_param_destroy(struct model_param *r); - void model_param_add(struct model *m, const char *name, enum model_param_direction dir, enum model_param_type type); -char * xsg_get_info(struct xsg_model *x, const char *key); - /** Read a model parameter. * * Note: the data type of the register is taken into account. @@ -97,4 +114,10 @@ int model_param_read(struct model_param *p, double *v); */ int model_param_write(struct model_param *p, double v); +/** Read the XSG model information from ROM */ +int model_xsg_map_read(void *offset, void *dst, size_t len); + +/** Parse binary model information from ROM */ +int model_xsg_map_parse(uint32_t *map, size_t len, struct list *parameters, struct list *infos); + #endif /* _MODEL_H_ */ \ No newline at end of file diff --git a/include/villas/fpga/xsg.h b/include/villas/fpga/xsg.h deleted file mode 100644 index 0b5f8a31a..000000000 --- a/include/villas/fpga/xsg.h +++ /dev/null @@ -1,54 +0,0 @@ -/** Interface to Xilinx System Generator Models via VILLASfpga - * - * @file - * @author Steffen Vogel - * @copyright 2016, Steffen Vogel - * This file is part of S2SS. All Rights Reserved. Proprietary and confidential. - * Unauthorized copying of this file, via any medium is strictly prohibited. - *********************************************************************************/ - -#ifndef _XSG_H_ -#define _XSG_H_ - -#define XSG_MAGIC 0xDEADBABE - -/* Forward declaration */ -struct model; - -enum xsg_block_type { - XSG_BLOCK_GATEWAY_IN = 0x1000, - XSG_BLOCK_GATEWAY_OUT = 0x1001, - XSG_BLOCK_INFO = 0x2000 -}; - -struct xsg_model { - int version; - - char *map; - size_t maplen; - - struct list infos; -}; - -struct xsg_info { - char *field; - char *value; -}; - -int xsg_init_from_map(struct model *m); - -int xsg_init_from_xml(struct model *m, const char *xml); - -void xsg_destroy(struct xsg_model *x); - -void xsg_dump(struct xsg_model *x); - -void xsg_info_destroy(struct xsg_info *i); - -/** Read the XSG model information from ROM */ -int xsg_map_read(void *offset, void *dst, size_t len); - -/** Parse binary model information from ROM */ -int xsg_map_parse(uint32_t *map, size_t len, struct list *parameters, struct list *infos); - -#endif /* _XSG_H_ */ \ No newline at end of file diff --git a/include/villas/nodes/vfpga.h b/include/villas/nodes/vfpga.h index 343d6ae80..137614b2e 100644 --- a/include/villas/nodes/vfpga.h +++ b/include/villas/nodes/vfpga.h @@ -17,53 +17,36 @@ #ifndef _VFPGA_H_ #define _VFPGA_H_ -#include -#include -#include -#include -#include - #include "kernel/vfio.h" -#include "fpga/switch.h" -#include "fpga/model.h" -#include "fpga/dma.h" -#include "fpga/fifo.h" -#include "fpga/rtds_axis.h" - #include #include "nodes/vfpga.h" #include "node.h" #include "list.h" +#define BASEADDR_HOST 0x8000000 #define VFPGA_BAR 0 /**< The Base Address Register which is mmap()ed to the User Space */ struct vfpga { - struct pci_filter filter; - - enum { - vfpga_MODE_FIFO, - vfpga_MODE_DMA - } mode; - - int features; - - struct list models; /**< List of XSG and HLS model blocks on FPGA. */ + struct pci_filter filter; /**< Filter for libpci with device id & slot */ struct vfio_dev vd; /**< VFIO device handle. */ - /* Xilinx IP blocks */ - XTmrCtr tmr; - XLlFifo fifo; - XAxiDma dma_simple; - XAxiDma dma_sg; - XAxis_Switch sw; + int reset; /**< Reset VILLASfpga during startup? */ + + struct list ips; /**< List of IP components on FPGA. */ char *map; /**< PCI BAR0 mapping for register access */ char *dma; /**< DMA mapped memory */ - + size_t maplen; size_t dmalen; + + /** Base addresses for internal IP blocks */ + struct { + uintptr_t reset; + uintptr_t intc; + } baseaddr; }; /** @see node_vtable::init */ @@ -73,7 +56,9 @@ int vfpga_init(int argc, char * argv[], config_setting_t *cfg); int vfpga_deinit(); /** @see node_vtable::parse */ -int vfpga_parse(struct node *n, config_setting_t *cfg); +int vfpga_parse_node(struct node *n, config_setting_t *cfg); + +int vfpga_parse(struct vfpga *v, int argc, char * argv[], config_setting_t *cfg); /** @see node_vtable::print */ char * vfpga_print(struct node *n); @@ -90,14 +75,13 @@ int vfpga_read(struct node *n, struct sample *smps[], unsigned cnt); /** @see node_vtable::write */ int vfpga_write(struct node *n, struct sample *smps[], unsigned cnt); -////////// - -int vfpga_init2(struct vfpga *f, struct vfio_container *vc, struct pci_dev *pdev); - -int vfpga_deinit2(struct vfpga *f); +/** Get pointer to internal VILLASfpga datastructure */ +struct vfpga * vfpga_get(); +/** Reset VILLASfpga */ int vfpga_reset(struct vfpga *f); +/** Dump some details about the fpga card */ void vfpga_dump(struct vfpga *f); #endif /** _VFPGA_H_ @} */ diff --git a/lib/cfg.c b/lib/cfg.c index 53798f96d..ec9535f6f 100644 --- a/lib/cfg.c +++ b/lib/cfg.c @@ -20,16 +20,18 @@ int config_parse(const char *filename, config_t *cfg, struct settings *set, struct list *nodes, struct list *paths) { - char *filename_cpy = strdup(filename); - char *include_dir = dirname(filename_cpy); + int ret; + char *filename_cpy, include_dir; + + filename_cpy = strdup(filename); + include_dir = dirname(filename_cpy); + free(filename_cpy); /* Setup libconfig */ config_set_auto_convert(cfg, 1); config_set_include_dir(cfg, include_dir); - free(filename_cpy); - - int ret = strcmp("-", filename) ? config_read_file(cfg, filename) : config_read(cfg, stdin); + ret = strcmp("-", filename) ? config_read_file(cfg, filename) : config_read(cfg, stdin); if (ret != CONFIG_TRUE) { error("Failed to parse configuration: %s in %s:%d", config_error_text(cfg), diff --git a/lib/fpga/model.c b/lib/fpga/model.c index 855b2ebbe..518018da3 100644 --- a/lib/fpga/model.c +++ b/lib/fpga/model.c @@ -15,20 +15,40 @@ #include #include "fpga/model.h" -#include "fpga/xsg.h" -int model_init(struct model *m, const char *xml) +int model_init(struct model *m, uintptr_t baseaddr) { - /** @todo: parse XML model descriptions */ + int ret; + + /** @todo: Currently only XSG models are supported */ list_init(&m->parameters); + list_init(&m->infos); + + ret = model_init_from_xsg_map(m, baseaddr); + if (ret) + error("Failed to init XSG model: %d", ret); return 0; } +static void model_param_destroy(struct model_param *p) +{ + free(p->name); +} + +static void model_info_destroy(struct model_info *i) +{ + free(i->field); + free(i->value); +} + void model_destroy(struct model *m) { list_destroy(&m->parameters, (dtor_cb_t) model_param_destroy, true); + list_destroy(&m->infos, (dtor_cb_t) model_info_destroy, true); + + free(m->map); } void model_dump(struct model *m) @@ -37,13 +57,13 @@ void model_dump(struct model *m) const char *model_types[] = { "HLS", "XSG" }; const char *parameter_dirs[] = { "In", "Out", "In/Out" }; - info("Model: %s (baseaddr=%p, type=%s)", m->name, m->baseaddr, model_types[m->type]); + info("Model: (type=%s)", model_types[m->type]); { INDENT info("Parameters:"); list_foreach(struct model_param *p, &m->parameters) { INDENT if (p->direction == MODEL_PARAM_IN) - info("%#x: %s (%s) = %.3f %s %u", + info("%#jx: %s (%s) = %.3f %s %u", p->offset, p->name, parameter_dirs[p->direction], @@ -52,27 +72,23 @@ void model_dump(struct model *m) p->binpt ); else if (p->direction == MODEL_PARAM_OUT) - info("%#x: %s (%s)", + info("%#jx: %s (%s)", p->offset, p->name, parameter_dirs[p->direction] ); } - switch (m->type) { - case MODEL_TYPE_HLS: - //hls_dump(&m->hls); - break; - case MODEL_TYPE_XSG: - xsg_dump(&m->xsg); - break; + info("Infos:"); + list_foreach(struct model_info *i, &m->infos) { INDENT + info("%s: %s", i->field, i->value); } } } int model_param_read(struct model_param *p, double *v) { - union model_param_value *ptr = p->offset + p->model->baseaddr; + union model_param_value *ptr = p->offset;// TODO: + p->model->baseaddr; switch (p->type) { case MODEL_PARAM_TYPE_UFIX: @@ -96,7 +112,7 @@ int model_param_read(struct model_param *p, double *v) int model_param_write(struct model_param *p, double v) { - union model_param_value *ptr = p->offset + p->model->baseaddr; + union model_param_value *ptr = p->offset;// TODO: + p->model->baseaddr; switch (p->type) { case MODEL_PARAM_TYPE_UFIX: @@ -130,7 +146,118 @@ void model_param_add(struct model *m, const char *name, enum model_param_directi list_push(&m->parameters, p); } -void model_param_destroy(struct model_param *p) +int model_init_from_xsg_map(struct model *m, uintptr_t baseaddr) { - free(p->name); + int ret; + + list_init(&m->infos); + + m->map = alloc(1024); + m->maplen = model_xsg_map_read(baseaddr, m->map, 1024 / 4); + if (m->maplen < 0) + return -1; + + debug(5, "XSG: memory map length = %#zx", m->maplen); + + ret = model_xsg_map_parse((uint32_t *) m->map, m->maplen, &m->parameters, &m->infos); + if (ret) + return -2; + + return 0; } + +static uint32_t model_xsg_map_checksum(uint32_t *map, size_t len) +{ + uint32_t chks = 0; + for (int i = 2; i < len-1; i++) + chks += map[i]; + + return chks; /* moduluo 2^32 because of overflow */ +} + +int model_xsg_map_parse(uint32_t *map, size_t len, struct list *parameters, struct list *infos) +{ +#define copy_string(off) strndup((char *) (data + (off)), (length - (off)) * 4); + + struct model_param *p; + struct model_info *i; + int j; + + if (map[0] != XSG_MAGIC) + error("Invalid magic: %#x", map[0]); + + for (j = 2; j < len-1;) { + uint16_t type = map[j] & 0xFFFF; + uint16_t length = map[j] >> 16; + uint32_t *data = &map[j+1]; + + switch (type) { + case XSG_BLOCK_GATEWAY_IN: + case XSG_BLOCK_GATEWAY_OUT: + if (length < 4) + break; /* block is to small to describe a gateway */ + + p = alloc(sizeof(*p)); + + p->name = copy_string(3); + p->default_value.flt = *((float *) &data[1]); + p->offset = data[2]; + p->direction = type & 0x1; + p->type = (data[0] >> 0) & 0xFF; + p->binpt = (data[0] >> 8) & 0xFF; + + list_push(parameters, p); + break; + + case XSG_BLOCK_INFO: + i = alloc(sizeof(struct model_info)); + + i->field = copy_string(0); + i->value = copy_string((int) ceil((double) (strlen(i->field) + 1) / 4)) + + list_push(infos, i); + break; + + default: + warn("Unknown block type: %#06x", type); + } + + j += length + 1; + } + + if (model_xsg_map_checksum(map, len) != map[j]) + error("XSG: Invalid checksum"); + + return 0; + +#undef copy_string +} + +int model_xsg_map_read(void *baseaddr, void *dst, size_t len) +{ +#define get_word(a) ({ *addr = a; *data; }) + + volatile uint32_t *addr = baseaddr + 0x00; + volatile uint32_t *data = baseaddr + 0x04; + + uint32_t *map = dst; + uint32_t maplen, magic; + + /* Check start DW */ + magic = get_word(0); + if (magic != XSG_MAGIC) + return -1; + + maplen = get_word(1); + if (maplen < 3) + return -2; + + /* Read Data */ + int i; + for (i = 0; i < MIN(maplen, len); i++) + map[i] = get_word(i); + + return i; + +#undef get_word +} \ No newline at end of file diff --git a/lib/fpga/xsg.c b/lib/fpga/xsg.c deleted file mode 100644 index 61cd62b66..000000000 --- a/lib/fpga/xsg.c +++ /dev/null @@ -1,176 +0,0 @@ -/** Interface to Xilinx System Generator Models via VILLASfpga - * - * @author Steffen Vogel - * @copyright 2016, Steffen Vogel - * This file is part of VILLASnode. All Rights Reserved. Proprietary and confidential. - * Unauthorized copying of this file, via any medium is strictly prohibited. - *********************************************************************************/ - -#include -#include -#include -#include - -#include -#include -#include - -#include "fpga/model.h" -#include "fpga/xsg.h" -#include "utils.h" - -int xsg_init_from_map(struct model *m) -{ - int ret; - struct xsg_model *x = &m->xsg; - - list_init(&x->infos); - - x->map = alloc(1024); - - x->maplen = xsg_map_read(m->baseaddr, x->map, 1024 / 4); - if (x->maplen < 0) - return -1; - - debug(5, "XSG: memory map length = %#zx", x->maplen); - - ret = xsg_map_parse((uint32_t *) x->map, x->maplen, &m->parameters, &x->infos); - if (ret) - return -2; - - m->name = xsg_get_info(x, "Name"); - - return 0; -} - -int xsg_init_from_xml(struct model *m, const char *xml) -{ - /** @todo: parse Simulink .mdl file here (XML based) */ - - return -1; -} - -void xsg_destroy(struct xsg_model *x) -{ - list_destroy(&x->infos, (dtor_cb_t) xsg_info_destroy, true); - - free(x->map); -} - -char * xsg_get_info(struct xsg_model *x, const char *key) -{ - struct xsg_info *info = list_lookup(&x->infos, key); - - return info ? info->value : NULL; -} - -void xsg_info_destroy(struct xsg_info *i) -{ - free(i->field); - free(i->value); -} - -void xsg_dump(struct xsg_model *x) -{ - info("XSG Infos:"); - list_foreach(struct xsg_info *i, &x->infos) { INDENT - info("%s: %s", i->field, i->value); - } -} - -static uint32_t xsg_map_checksum(uint32_t *map, size_t len) -{ - uint32_t chks = 0; - for (int i = 2; i < len-1; i++) - chks += map[i]; - - return chks; /* moduluo 2^32 because of overflow */ -} - -int xsg_map_parse(uint32_t *map, size_t len, struct list *parameters, struct list *infos) -{ -#define copy_string(off) strndup((char *) (data + (off)), (length - (off)) * 4); - - struct model_param *p; - struct xsg_info *info; - - int i; - - if (map[0] != XSG_MAGIC) - error("Invalid magic: %#x", map[0]); - - for (i = 2; i < len-1;) { - uint16_t type = map[i] & 0xFFFF; - uint16_t length = map[i] >> 16; - uint32_t *data = &map[i+1]; - - switch (type) { - case XSG_BLOCK_GATEWAY_IN: - case XSG_BLOCK_GATEWAY_OUT: - if (length < 4) - break; /* block is to small to describe a gateway */ - - p = alloc(sizeof(*p)); - - p->name = copy_string(3); - p->default_value.flt = *((float *) &data[1]); - p->offset = data[2]; - p->direction = type & 0x1; - p->type = (data[0] >> 0) & 0xFF; - p->binpt = (data[0] >> 8) & 0xFF; - - list_push(parameters, p); - break; - - case XSG_BLOCK_INFO: - info = alloc(sizeof(*info)); - - info->field = copy_string(0); - info->value = copy_string((int) ceil((double) (strlen(info->field) + 1) / 4)) - - list_push(infos, info); - break; - - default: - warn("Unknown block type: %#06x", type); - } - - i += length + 1; - } - - if (xsg_map_checksum(map, len) != map[i]) - error("XSG: Invalid checksum"); - - return 0; - -#undef copy_string -} - -int xsg_map_read(void *baseaddr, void *dst, size_t len) -{ -#define get_word(a) ({ *addr = a; *data; }) - - volatile uint32_t *addr = baseaddr + 0x00; - volatile uint32_t *data = baseaddr + 0x04; - - uint32_t *map = dst; - uint32_t maplen, magic; - - /* Check start DW */ - magic = get_word(0); - if (magic != XSG_MAGIC) - return -1; - - maplen = get_word(1); - if (maplen < 3) - return -2; - - /* Read Data */ - int i; - for (i = 0; i < MIN(maplen, len); i++) - map[i] = get_word(i); - - return i; - -#undef get_word -} \ No newline at end of file diff --git a/lib/nodes/vfpga.c b/lib/nodes/vfpga.c index 0c78e1da0..f1266e13d 100644 --- a/lib/nodes/vfpga.c +++ b/lib/nodes/vfpga.c @@ -7,7 +7,7 @@ *********************************************************************************/ #include -#include +#include #include #include #include @@ -17,28 +17,202 @@ #include "nodes/vfpga.h" -#include "config.h" +#include "fpga/ip.h" + #include "config-fpga.h" #include "utils.h" #include "timing.h" +struct vfpga fpga; +struct vfio_container vc; static struct pci_access *pacc; typedef void(*log_cb_t)(char *, ...); -int vfpga_init2(struct vfpga *f, struct vfio_container *vc, struct pci_dev *pdev) +int vfpga_reset(struct vfpga *f) { int ret; + char state[4096]; - list_init(&f->models); + /* Save current state of PCI configuration space */ + ret = pread(f->vd.fd, state, sizeof(state), (off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40); + if (ret != sizeof(state)) + return -1; + + uint32_t *rst_reg = (uint32_t *) (f->map + f->baseaddr.reset); + + info("Reset fpga"); + rst_reg[0] = 1; + + usleep(100000); + + /* Restore previous state of PCI configuration space */ + ret = pwrite(f->vd.fd, state, sizeof(state), (off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40); + if (ret != sizeof(state)) + return -1; + + info("Reset status = %#x", *rst_reg); + + return 0; +} + +void vfpga_dump(struct vfpga *f) +{ + char namebuf[128]; + char *name; + + name = pci_lookup_name(pacc, namebuf, sizeof(namebuf), PCI_LOOKUP_DEVICE, fpga.vd.pdev->vendor_id, fpga.vd.pdev->device_id); + pci_fill_info(fpga.vd.pdev, PCI_FILL_IDENT | PCI_FILL_BASES | PCI_FILL_CLASS); /* Fill in header info we need */ + + info("VILLASfpga card: %s", name); + { INDENT + info("Slot: %04x:%02x:%02x.%d", fpga.vd.pdev->domain, fpga.vd.pdev->bus, fpga.vd.pdev->dev, fpga.vd.pdev->func); + info("Vendor ID: %04x", fpga.vd.pdev->vendor_id); + info("Device ID: %04x", fpga.vd.pdev->device_id); + info("Class ID: %04x", fpga.vd.pdev->device_class); + + info("BAR0 mapped at %p", fpga.map); + info("DMA mapped at %p", fpga.dma); + + info("IP blocks:"); + list_foreach(struct ip *i, &f->ips) { INDENT + ip_dump(i); + } + } + + vfio_dump(fpga.vd.group->container); +} + +struct vfpga * vfpga_get() +{ + return &fpga; +} + +static void vfpga_debug(char *msg, ...) { + va_list ap; + + va_start(ap, msg); + log_vprint(LOG_LVL_DEBUG, msg, ap); + va_end(ap); +} + +int vfpga_parse(struct vfpga *v, int argc, char * argv[], config_setting_t *cfg) +{ + int ret; + const char *slot, *id, *err; + config_setting_t *cfg_fpga, *cfg_ips, *cfg_slot, *cfg_id; + + /* Default values */ + v->filter.vendor = PCI_VID_XILINX; + v->filter.device = PCI_PID_VFPGA; + v->reset = false; + + cfg_fpga = config_setting_get_member(cfg, "fpga"); + if (!cfg_fpga) + cerror(cfg, "Config file is missing VILLASfpga configuration"); + + config_setting_lookup_bool(cfg_fpga, "reset", &v->reset); + + cfg_slot = config_setting_get_member(cfg_fpga, "slot"); + if (cfg_slot) { + slot = config_setting_get_string(cfg_slot); + if (slot) { + err = pci_filter_parse_slot(&v->filter, (char*) slot); + if (err) + cerror(cfg_slot, "%s", err); + } + else + cerror(cfg_slot, "Invalid slot format"); + } + + cfg_id = config_setting_get_member(cfg_fpga, "id"); + if (cfg_id) { + id = config_setting_get_string(cfg_id); + if (id) { + err = pci_filter_parse_id(&v->filter, (char*) id); + if (err) + cerror(cfg_id, "%s", err); + } + else + cerror(cfg_slot, "Invalid id format"); + } - f->features = 0; + cfg_ips = config_setting_get_member(cfg_fpga, "ips"); + if (!cfg_ips) + cerror(cfg_fpga, "FPGA configuration is missing ips section"); + + for (int i = 0; i < config_setting_length(cfg_ips); i++) { + struct ip ip; + config_setting_t *cfg_ip = config_setting_get_elem(cfg_ips, i); + + ret = ip_parse(&ip, cfg_ip); + if (ret) + cerror(cfg_ip, "Failed to parse VILLASfpga ip"); + + list_push(&v->ips, memdup(&ip, sizeof(struct ip))); + } + + return 0; +} + +struct pci_access * vfpga_init_pci() +{ + struct pci_access *pacc; + + pacc = pci_alloc(); /* Get the pci_access structure */ + if (!pacc) + error("Failed to allocate PCI access structure"); + + pci_init(pacc); /* Initialize the PCI library */ + pci_scan_bus(pacc); /* We want to get the list of devices */ + + pacc->error = (log_cb_t) error; /* Replace logging and debug functions */ + pacc->warning = (log_cb_t) warn; + pacc->debug = vfpga_debug; + + pci_scan_bus(pacc); /* We want to get the list of devices */ + + return pacc; +} + +int vfpga_init(int argc, char * argv[], config_setting_t *cfg) +{ + int ret; + struct pci_access *pacc; + struct pci_dev *pdev; + struct vfpga *f; + + /* For now we only support a single VILALSfpga card */ + f = vfpga_get(); + + /* Some hardcoded addresses for internal IP blocks */ + f->baseaddr.reset = 0x2000; + f->baseaddr.intc = 0x5000; + + pacc = vfpga_init_pci(); + + pci_filter_init(pacc, &f->filter); + list_init(&f->ips); + + ret = vfpga_parse(f, argc, argv, cfg); + if (ret) + cerror(cfg, "Failed to parse VILLASfpga config"); + + /* Search for fpga card */ + pdev = pci_find_device(pacc, &f->filter); + if (!pdev) + error("Failed to find PCI device"); if (pdev->vendor_id != PCI_VID_XILINX) error("This is not a Xilinx FPGA board!"); + /* Get VFIO handles and details */ + ret = vfio_init(&vc); + if (ret) + serror("Failed to initialize VFIO"); + /* Attach PCIe card to VFIO container */ - ret = vfio_pci_attach(&f->vd, vc, pdev); + ret = vfio_pci_attach(&f->vd, &vc, pdev); if (ret) error("Failed to attach VFIO device"); @@ -64,223 +238,44 @@ int vfpga_init2(struct vfpga *f, struct vfio_container *vc, struct pci_dev *pdev if (ret) serror("Failed to enable PCI device"); -#if 1 /* Trigger internal reset of fpga card / PCIe endpoint */ - ret = vfpga_reset(f); - if (ret) - serror("Failed to reset fpga card"); -#endif - - /* Reset / detect PCI device */ - ret = vfio_pci_reset(&f->vd); - if (ret) - serror("Failed to reset PCI device"); - - /* Initialize Interrupt controller */ - XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IMR_OFFSET, 0); - XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IAR_OFFSET, 0xFFFFFFFF); /* Acknowlege all pending IRQs */ - usleep(1000); - - /* Setup Vectors */ - for (int i = 0; i < 16; i++) - XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IVAR_OFFSET + i * sizeof(uint32_t), i); - - XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IMR_OFFSET, 0xFFFFFFFF); /* Use fast acknowlegement for all IRQs */ - XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, 0x00000000); - XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_MER_OFFSET, XIN_INT_HARDWARE_ENABLE_MASK | XIN_INT_MASTER_ENABLE_MASK); - -#ifdef HAS_SWITCH - /* Initialize AXI4-Sream crossbar switch */ - ret = switch_init(&f->sw, f->map + BASEADDR_SWITCH, AXI_SWITCH_NUM, AXI_SWITCH_NUM); - if (ret) - error("Failed to initialize switch"); -#endif - -#ifdef HAS_XSG - /* Initialize System Generator model */ - struct model *m = alloc(sizeof(struct model)); - m->baseaddr = f->map + BASEADDR_XSG; - m->type = MODEL_TYPE_XSG; - - ret = model_init(m, NULL); - if (ret) - error("Failed to init model: %d", ret); - - ret = xsg_init_from_map(m); - if (ret) - error("Failed to init XSG model: %d", ret); - - list_push(&f->models, m); -#endif - -#ifdef HAS_FIFO_MM - /* Initialize AXI4-Stream Memory Mapped FIFO */ - ret = fifo_init(&f->fifo, f->map + BASEADDR_FIFO_MM, f->map + BASEADDR_FIFO_MM_AXI4); - if (ret) - error("Failed to initialize FIFO"); -#endif - -#ifdef HAS_DMA_SG - /* Initialize Scatter Gather DMA controller */ - struct dma_mem bd = { - .base_virt = (uintptr_t) f->map + BASEADDR_BRAM, - .base_phys = BASEADDR_BRAM, - .len = 0x2000 - }; - struct dma_mem rx = { - .base_virt = (uintptr_t) f->dma, - .base_phys = BASEADDR_HOST, - .len = 0x10000 - }; - - ret = dma_init(&f->dma_sg, f->map + BASEADDR_DMA_SG, DMA_MODE_SG, &bd, &rx, sizeof(uint32_t) * 64); - if (ret) - return -1; -#endif - -#ifdef HAS_DMA_SIMPLE - /* Initialize Simple DMA controller */ - ret = dma_init(&f->dma_simple, f->map + BASEADDR_DMA_SIMPLE, DMA_MODE_SIMPLE, 0, 0, 0); - if (ret) - return -1; -#endif - -#ifdef HAS_TIMER - /* Initialize timer */ - XTmrCtr_Config tmr_cfg = { - .BaseAddress = (uintptr_t) f->map + BASEADDR_TIMER, - .SysClockFreqHz = AXI_HZ - }; - - XTmrCtr_CfgInitialize(&f->tmr, &tmr_cfg, (uintptr_t) f->map + BASEADDR_TIMER); - XTmrCtr_InitHw(&f->tmr); -#endif - - return 0; -} - -int vfpga_deinit2(struct vfpga *f) -{ - list_destroy(&f->models, (dtor_cb_t) model_destroy, true); - - return 0; -} - -int vfpga_reset(struct vfpga *f) -{ - int ret; - char state[4096]; - - /* Save current state of PCI configuration space */ - ret = pread(f->vd.fd, state, sizeof(state), (off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40); - if (ret != sizeof(state)) - return -1; - - uint32_t *rst_reg = (uint32_t *) (f->map + BASEADDR_RESET); - - info("Reset fpga"); - rst_reg[0] = 1; - - usleep(100000); - - /* Restore previous state of PCI configuration space */ - ret = pwrite(f->vd.fd, state, sizeof(state), (off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40); - if (ret != sizeof(state)) - return -1; - - info("Reset status = %#x", *rst_reg); - - return 0; -} - -/* Dump some details about the fpga card */ -void vfpga_dump(struct vfpga *f) -{ - char namebuf[128]; - char *name; - - name = pci_lookup_name(pacc, namebuf, sizeof(namebuf), PCI_LOOKUP_DEVICE, f->vd.pdev->vendor_id, f->vd.pdev->device_id); - pci_fill_info(f->vd.pdev, PCI_FILL_IDENT | PCI_FILL_BASES | PCI_FILL_CLASS); /* Fill in header info we need */ - - info("Found fpga card: %s", name); - { INDENT - info("slot=%04x:%02x:%02x.%d", f->vd.pdev->domain, f->vd.pdev->bus, f->vd.pdev->dev, f->vd.pdev->func); - info("vendor=%04x", f->vd.pdev->vendor_id); - info("device=%04x", f->vd.pdev->device_id); - info("class=%04x", f->vd.pdev->device_class); + if (f->reset) { + ret = vfpga_reset(f); + if (ret) + serror("Failed to reset fpga card"); + + /* Reset / detect PCI device */ + ret = vfio_pci_reset(&f->vd); + if (ret) + serror("Failed to reset PCI device"); } - info("BAR0 mapped at %p", f->map); - vfio_dump(f->vd.group->container); -} - -static void vfpga_debug(char *msg, ...) { - va_list ap; - - va_start(ap, msg); - log_vprint(LOG_LVL_DEBUG, msg, ap); - va_end(ap); -} - -int vfpga_init(int argc, char * argv[], config_setting_t *cfg) -{ - if (getuid() != 0) - error("The vfpga node-type requires superuser privileges!"); - - pacc = pci_alloc(); /* Get the pci_access structure */ - if (!pacc) - error("Failed to allocate PCI access structure"); - - pci_init(pacc); /* Initialize the PCI library */ - - pacc->error = (log_cb_t) error; /* Replace logging and debug functions */ - pacc->warning = (log_cb_t) warn; - pacc->debug = vfpga_debug; - - pci_scan_bus(pacc); /* We want to get the list of devices */ + list_foreach(struct ip *c, &f->ips) { + c->card = f; + ip_init(c); + } return 0; } int vfpga_deinit() { + int ret; + + list_destroy(&fpga.ips, (dtor_cb_t) ip_destroy, true); + pci_cleanup(pacc); + ret = vfio_destroy(&vc); + if (ret) + error("Failed to deinitialize VFIO module"); + return 0; } -int vfpga_parse(struct node *n, config_setting_t *cfg) +int vfpga_parse_node(struct node *n, config_setting_t *cfg) { - struct vfpga *v = n->_vd; - - const char *slot, *id, *err; - config_setting_t *cfg_slot, *cfg_id; - - pci_filter_init(NULL, &v->filter); - - cfg_slot = config_setting_get_member(cfg, "slot"); - if (cfg_slot) { - slot = config_setting_get_string(cfg_slot); - if (slot) { - err = pci_filter_parse_slot(&v->filter, (char*) slot); - if (err) - cerror(cfg_slot, "%s", err); - } - else - cerror(cfg_slot, "Invalid slot format"); - } - - cfg_id = config_setting_get_member(cfg, "id"); - if (cfg_id) { - id = config_setting_get_string(cfg_id); - if (id) { - err = pci_filter_parse_id(&v->filter, (char*) id); - if (err) - cerror(cfg_id, "%s", err); - } - else - cerror(cfg_slot, "Invalid id format"); - } +// struct vfpga *v = n->_vd; return 0; } @@ -339,8 +334,8 @@ int vfpga_write(struct node *n, struct sample *smps[], unsigned cnt) static struct node_type vt = { .name = "vfpga", .description = "VILLASfpga PCIe card (libpci)", - .vectorize = 0, - .parse = vfpga_parse, + .vectorize = 1, + .parse = vfpga_parse_node, .print = vfpga_print, .open = vfpga_open, .close = vfpga_close, diff --git a/src/fpga-bench.c b/src/fpga-bench.c index 31e7df764..d86b80b0a 100644 --- a/src/fpga-bench.c +++ b/src/fpga-bench.c @@ -31,7 +31,7 @@ enum benchmarks { MAX_BENCHS }; -static int bench_run(struct vfpga *f, int polling, int bench) +int fpga_bench(struct vfpga *f, int polling, int bench) { struct hist hist; uint64_t start, stop; @@ -87,7 +87,7 @@ static int bench_run(struct vfpga *f, int polling, int bench) hist_destroy(&hist); } -int bench_dm(struct vfpga *f) +int fpga_bench_dm(struct vfpga *f) { int irq_fifo, irq_dma_mm2s, irq_dma_s2mm; int polling = true; @@ -106,7 +106,7 @@ int bench_dm(struct vfpga *f) return 0; } -int bench_memcpy(struct vfpga *f) +int fpga_bench_memcpy(struct vfpga *f) { uint64_t start, end, total = 0; diff --git a/src/fpga-tests.c b/src/fpga-tests.c index 3384e9d1e..c2817c8df 100644 --- a/src/fpga-tests.c +++ b/src/fpga-tests.c @@ -13,10 +13,19 @@ #include #include +#include +#include +#include +#include +#include + #include #include -#include "fpga-tests.h" +#include + +//#include + #include "config.h" #include "config-fpga.h" @@ -26,83 +35,140 @@ #define CPU_HZ 3392389000 -int test_intc(struct vfpga *f) +/* Declarations */ +int fpga_test_intc(struct vfpga *f); +int fpga_test_timer(struct vfpga *f); +int fpga_test_fifo(struct vfpga *f); +int fpga_test_dma(struct vfpga *f); +int fpga_test_xsg(struct vfpga *f); +int fpga_test_rtds_rtt(struct vfpga *f); +int fpga_test_rtds_cbuilder(struct vfpga *f); + +int fpga_tests(struct vfpga *f) +{ + int ret; + + struct { + const char *name; + int (*func)(struct vfpga *f); + } tests[] = { + { "Interrupt Controller", fpga_test_intc }, + { "Timer Counter", fpga_test_timer }, + { "FIFO", fpga_test_fifo }, + { "DMA", fpga_test_dma }, + { "XSG: multiply_add", fpga_test_xsg }, + { "RTDS: RTT Test", fpga_test_rtds_rtt }, + { "RTDS: CBuilder", fpga_test_rtds_cbuilder } + }; + + for (int i = 0; i < ARRAY_LEN(tests); i++) { + ret = tests[i].func(f); + + info("%s: %s", tests[i].name, (ret == 0) ? GRN("passed") : RED("failed")); + } + + return 0; +} + +int fpga_test_intc(struct vfpga *f) { uint32_t ier, isr; /* Save old IER */ - ier = XIntc_In32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET); - XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier | 0xFF00); + ier = XIntc_In32(f->baseaddr.intc + XIN_IER_OFFSET); + XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier | 0xFF00); - XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_ISR_OFFSET, 0xFF00); + XIntc_Out32(f->baseaddr.intc + XIN_ISR_OFFSET, 0xFF00); debug(3, "Wait for 8 SW triggerd interrupts"); for (int i = 0; i < 8; i++) wait_irq(f->vd.msi_efds[i+8]); - isr = XIntc_In32((uintptr_t) f->map + BASEADDR_INTC + XIN_ISR_OFFSET); + isr = XIntc_In32(f->baseaddr.intc + XIN_ISR_OFFSET); /* Restore IER */ - XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier); + XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier); return (isr & 0xFF00) ? -1 : 0; /* ISR should get cleared by MSI_Grant_signal */ } -#ifdef HAS_XSG -int test_xsg(struct vfpga *f, const char *name) +int fpga_test_xsg(struct vfpga *f) { - struct model *m; + struct ip *xsg, *dma, *sw; + + xsg = ip_vlnv_lookup(&f->ips, NULL, "xsg", "xsg_multiply_add", NULL); + sw = ip_vlnv_lookup(&f->ips, "xilinx.com", "ip", "axis_interconnect", NULL); + dma = ip_vlnv_lookup(&f->ips, "xilinx.com", "ip", "axi_dma", NULL); - m = list_lookup(&f->models, name); - if (m) { - model_dump(m); + /* Check if required IP is available on FPGA */ + if (!dma || !xsg || !dma) + return -1; - list_foreach(struct model_param *p, &m->parameters) { - if (p->direction == MODEL_PARAM_IN) { - model_param_write(p, p->default_value.flt); - info("Param '%s' updated to: %f", p->name, p->default_value.flt); - } - } + ip_dump(xsg); - if (strcmp(m->name, "xsg_multiply_add") == 0) { - switch_connect(&f->sw, AXIS_SWITCH_DMA_SIMPLE, AXIS_SWITCH_XSG); - switch_connect(&f->sw, AXIS_SWITCH_XSG, AXIS_SWITCH_DMA_SIMPLE); - - float *src = (float *) f->dma; - float *dst = (float *) f->dma + TEST_LEN; - - for (int i = 0; i < 6; i++) - src[i] = 1.1 * (i+1); - - ssize_t len = dma_ping_pong(&f->dma_simple, virt_to_dma(src, f->dma), virt_to_dma(dst, f->dma), 6 * sizeof(float), -1); - if (len != 6 * sizeof(float)) - error("Failed to to ping pong DMA transfer: %zd", len); - - for (int i = 0; i < 6; i++) - info("src[%d] = %f", i, src[i]); - for (int i = 0; i < 6; i++) - info("dst[%d] = %f", i, dst[i]); + list_foreach(struct model_param *p, &xsg->model.parameters) { + if (p->direction == MODEL_PARAM_IN) { + model_param_write(p, p->default_value.flt); + info("Param '%s' updated to: %f", p->name, p->default_value.flt); } } - else - warn("There is no model named '%s'", name); + + switch_connect(&sw->sw, dma->port, xsg->port); + switch_connect(&sw->sw, xsg->port, dma->port); + + float *src = (float *) f->dma; + float *dst = (float *) f->dma + TEST_LEN; + + for (int i = 0; i < 6; i++) + src[i] = 1.1 * (i+1); + + ssize_t len = dma_ping_pong(&dma->dma, virt_to_dma(src, f->dma), virt_to_dma(dst, f->dma), 6 * sizeof(float), -1); + if (len != 6 * sizeof(float)) + error("Failed to to ping pong DMA transfer: %zd", len); + + for (int i = 0; i < 6; i++) + info("src[%d] = %f", i, src[i]); + for (int i = 0; i < 6; i++) + info("dst[%d] = %f", i, dst[i]); return 0; } -#endif -#ifdef HAS_FIFO_MM -int test_fifo(struct vfpga *f) +int fpga_test_hls_dft(struct vfpga *f) +{ + struct ip *hls, *sw, *dma; + + hls = ip_vlnv_lookup(&f->ips, NULL, "hls", NULL, NULL); + sw = ip_vlnv_lookup(&f->ips, "xilinx.com", "ip", "axis_interconnect", NULL); + dma = ip_vlnv_lookup(&f->ips, "xilinx.com", "ip", "axi_dma", NULL); + + /* Check if required IP is available on FPGA */ + if (!dma || !hls || !dma) + return -1; + + + return 0; +} + +int fpga_test_fifo(struct vfpga *f) { int ret; ssize_t len; - uintptr_t baseaddr = (uintptr_t) (f->map + BASEADDR_FIFO_MM); char src[255], dst[255]; + struct ip *fifo, *sw; uint32_t ier; + fifo = ip_vlnv_lookup(&f->ips, "xilinx.com", "ip", "axi_fifo_mm", NULL); + if (!fifo) + return -1; + + sw = ip_vlnv_lookup(&f->ips, "xilinx.com", "ip", "axis_interconnect", NULL); + if (!sw) + return -1; + /* Save old IER */ - ier = XIntc_In32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET); - XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier | (1 << MSI_FIFO_MM)); + ier = XIntc_In32(f->baseaddr.intc + XIN_IER_OFFSET); + XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier | (1 << fifo->irq)); /* Get some random data to compare */ memset(dst, 0, sizeof(dst)); @@ -110,38 +176,48 @@ int test_fifo(struct vfpga *f) if (ret) error("Failed to get random data"); - ret = switch_connect(&f->sw, AXIS_SWITCH_FIFO_MM, AXIS_SWITCH_FIFO_MM); + ret = switch_connect(&sw->sw, fifo->port, fifo->port); if (ret) error("Failed to configure switch"); - len = fifo_write(&f->fifo, (char *) src, sizeof(src), f->vd.msi_efds[MSI_FIFO_MM]); + len = fifo_write(&fifo->fifo_mm, (char *) src, sizeof(src), f->vd.msi_efds[fifo->irq]); if (len != sizeof(src)) error("Failed to send to FIFO"); - len = fifo_read(&f->fifo, (char *) dst, sizeof(dst), f->vd.msi_efds[MSI_FIFO_MM]); + len = fifo_read(&fifo->fifo_mm, (char *) dst, sizeof(dst), f->vd.msi_efds[fifo->irq]); if (len != sizeof(dst)) error("Failed to read from FIFO"); /* Restore IER */ - XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier); + XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier); /* Compare data */ return memcmp(src, dst, sizeof(src)); } -#endif -#if defined(HAS_DMA_SG) || defined(HAS_DMA_SIMPLE) -int test_dma(struct vfpga *f) +int fpga_test_dma(struct vfpga *f) { - int ret; + struct ip *dma, *sw; ssize_t len; + int ret; uint32_t ier; - /* Save old IER */ - ier = XIntc_In32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET); - XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier | (1 << MSI_DMA_MM2S) | (1 << MSI_DMA_S2MM)); + dma = ip_vlnv_lookup(&f->ips, "xilinx.com", "ip", "axi_dma", NULL); + if (!dma) + return -1; - ret = switch_connect(&f->sw, AXIS_SWITCH_DMA_SIMPLE, AXIS_SWITCH_DMA_SIMPLE); + sw = ip_vlnv_lookup(&f->ips, "xilinx.com", "ip", "axis_interconnect", NULL); + if (!sw) + return -1; + + int irq_mm2s = dma->irq; + int irq_s2mm = dma->irq + 1; + + /* Save old IER */ + ier = XIntc_In32(f->baseaddr.intc + XIN_IER_OFFSET); + XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier | (1 << irq_mm2s) | (1 << irq_s2mm)); + + ret = switch_connect(&sw->sw, dma->port, dma->port); if (ret) error("Failed to configure switch"); @@ -161,60 +237,62 @@ int test_dma(struct vfpga *f) #endif /* Get some random data to compare */ - char *src = f->dma; - char *dst = f->dma + TEST_LEN; + char *src = (char *) f->dma; + char *dst = (char *) f->dma + TEST_LEN; ret = read_random(src, TEST_LEN); if (ret) error("Failed to get random data"); #if 0 - len = dma_write(&f->dma_simple, virt_to_dma(src, f->dma), TEST_LEN, -1);//f->vd.msi_efds[MSI_DMA_MM2S]); + len = dma_write(&f->dma_simple, virt_to_dma(src, f->dma), TEST_LEN, -1);//f->vd.msi_efds[irq_mm2s]); if (len != TEST_LEN) error("Failed to send to DMAC: %zd", len); - len = dma_read(&f->dma_simple, virt_to_dma(dst, f->dma), TEST_LEN, -1);//f->vd.msi_efds[MSI_DMA_S2MM]); + len = dma_read(&f->dma_simple, virt_to_dma(dst, f->dma), TEST_LEN, -1);//f->vd.msi_efds[irq_s2mm]); if (len != TEST_LEN) error("Failed to send to DMAC: %zd", len); #else - len = dma_ping_pong(&f->dma_simple, virt_to_dma(src, f->dma), virt_to_dma(dst, f->dma), TEST_LEN, f->vd.msi_efds[MSI_DMA_S2MM]); + len = dma_ping_pong(&dma->dma, virt_to_dma(src, f->dma), virt_to_dma(dst, f->dma), TEST_LEN, f->vd.msi_efds[irq_s2mm]); if (len != TEST_LEN) error("Failed to to ping pong DMA transfer: %zd", len); #endif /* Restore IER */ - XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier); + XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier); /* Check received data */ return memcmp(src, dst, TEST_LEN); } -#endif -#ifdef HAS_TIMER -int test_timer(struct vfpga *f) +int fpga_test_timer(struct vfpga *f) { int ret; - bool success; + struct ip *timer; uint32_t ier; - /* Save old IER */ - ier = XIntc_In32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET); - XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier | (1 << MSI_TMRCTR0)); + timer = ip_vlnv_lookup(&f->ips, "xilinx.com", "ip", "axi_timer", NULL); + if (!timer) + return -1; - XTmrCtr_SetOptions(&f->tmr, 0, XTC_EXT_COMPARE_OPTION | XTC_DOWN_COUNT_OPTION); - XTmrCtr_SetResetValue(&f->tmr, 0, AXI_HZ / 125); - XTmrCtr_Start(&f->tmr, 0); + /* Save old IER */ + ier = XIntc_In32(f->baseaddr.intc + XIN_IER_OFFSET); + XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier | (1 << timer->irq)); + + XTmrCtr_SetOptions(&timer->timer, 0, XTC_EXT_COMPARE_OPTION | XTC_DOWN_COUNT_OPTION); + XTmrCtr_SetResetValue(&timer->timer, 0, AXI_HZ / 125); + XTmrCtr_Start(&timer->timer, 0); struct pollfd pfd = { - .fd = f->vd.msi_efds[MSI_TMRCTR0], + .fd = f->vd.msi_efds[timer->irq], .events = POLLIN }; ret = poll(&pfd, 1, 1000); if (ret == 1) { - uint64_t counter = wait_irq(f->vd.msi_efds[MSI_TMRCTR0]); + uint64_t counter = wait_irq(f->vd.msi_efds[timer->irq]); - info("Got IRQ: counter = %llu", counter); + info("Got IRQ: counter = %ju", counter); if (counter == 1) return 0; @@ -223,41 +301,51 @@ int test_timer(struct vfpga *f) } /* Restore IER */ - XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier); + XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier); return -1; } -#endif -#ifdef HAS_RTDS_AXIS -int test_rtds_cbuilder(struct vfpga *f) +int fpga_test_rtds_cbuilder(struct vfpga *f) { +#if 0 int ret; int values_tx = 1; int values_rx; uint32_t tsc = 0, ovfl = 0; ssize_t len; uint32_t ier; + struct ip *rtds, *dma; + + int irq_rtds_ovf = rtds->irq + 1; + + rtds = ip_vlnv_lookup(&f->ips, "acs.eonerc.rwth-aachen.de", "user", "rtds_axis", NULL); + if (!rtds) + return -1; + + dma = ip_vlnv_lookup(&f->ips, "xilinx.com", "ip", "axi_dma", NULL); + if (!dma) + return -1; /* Save old IER */ - ier = XIntc_In32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET); -// XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier | (1 << MSI_DMA_MM2S) | (1 << MSI_DMA_S2MM)); - XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier | (1 << MSI_RTDS_OVF)); + ier = XIntc_In32(f->baseaddr.intc + XIN_IER_OFFSET); +// XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier | (1 << MSI_DMA_MM2S) | (1 << MSI_DMA_S2MM)); + XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier | (1 << irq_rtds_ovf); /* Dump RTDS AXI Stream state */ - rtds_axis_dump(f->map + BASEADDR_RTDS); + rtds_axis_dump(f->map + rtds->baseaddr); /* Get some random data to compare */ float *data_rx = (uint32_t *) (f->dma); float *data_tx = (uint32_t *) (f->dma + 0x1000); /* Setup crossbar switch */ - switch_connect(&f->sw, AXIS_SWITCH_RTDS, AXIS_SWITCH_DMA_SIMPLE); - switch_connect(&f->sw, AXIS_SWITCH_DMA_SIMPLE, AXIS_SWITCH_RTDS); + switch_connect(&sw->sw, rtds->port, dma->port); + switch_connect(&sw->sw, dma->port, rtds->port); /* Disable blocking Overflow status */ - int flags = fcntl(f->vd.msi_efds[MSI_RTDS_OVF], F_GETFL, 0); - fcntl(f->vd.msi_efds[MSI_RTDS_OVF], F_SETFL, flags | O_NONBLOCK); + int flags = fcntl(f->vd.msi_efds[irq_rtds_ovf], F_GETFL, 0); + fcntl(f->vd.msi_efds[irq_rtds_ovf], F_SETFL, flags | O_NONBLOCK); /* Initialize CBuilder model */ double mdl_dt = rtds_axis_dt(f->map + BASEADDR_RTDS); @@ -279,9 +367,8 @@ int test_rtds_cbuilder(struct vfpga *f) break; /* Read data from RTDS */ - len = dma_read(&f->dma_simple, virt_to_dma(data_rx, f->dma), 0x1000, -1); - uint64_t start = rdtscp(); - XAxiDma_IntrAckIrq(&f->dma_simple, XAXIDMA_IRQ_IOC_MASK, XAXIDMA_DEVICE_TO_DMA); + len = dma_read(&dma->dma, virt_to_dma(data_rx, f->dma), 0x1000, -1); + XAxiDma_IntrAckIrq(&dma->dma, XAXIDMA_IRQ_IOC_MASK, XAXIDMA_DEVICE_TO_DMA); values_rx = len / sizeof(uint32_t); @@ -289,32 +376,30 @@ int test_rtds_cbuilder(struct vfpga *f) /* Run CBuilder model */ cbmodel_step(mdl_inputs, mdl_outputs); - + data_tx[0] = mdl_outputs[0]; /* cast to float */ /* Send data to RTDS */ - len = dma_write(&f->dma_simple, virt_to_dma(data_tx, f->dma), sizeof(uint32_t) * values_tx, -1); - XAxiDma_IntrAckIrq(&f->dma_simple, XAXIDMA_IRQ_IOC_MASK, XAXIDMA_DMA_TO_DEVICE); + len = dma_write(&dma->dma, virt_to_dma(data_tx, f->dma), sizeof(uint32_t) * values_tx, -1); + XAxiDma_IntrAckIrq(&dma->dma, XAXIDMA_IRQ_IOC_MASK, XAXIDMA_DMA_TO_DEVICE); } /* Restore IER */ - XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier); - + XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier); +#endif return 0; } -int test_rtds_rtt(struct vfpga *f) +int fpga_test_rtds_rtt(struct vfpga *f) { - int ret; - int values_tx = 64; - int values_rx; - uint32_t tsc = 0, ovfl = 0; +#if 0 + uint32_t tsc = 0; ssize_t len; uint32_t ier; /* Save old IER */ - ier = XIntc_In32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET); -// XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier | (1 << MSI_DMA_MM2S) | (1 << MSI_DMA_S2MM)); + ier = XIntc_In32(f->baseaddr.intc + XIN_IER_OFFSET); +// XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier | (1 << MSI_DMA_MM2S) | (1 << MSI_DMA_S2MM)); /* Dump RTDS AXI Stream state */ rtds_axis_dump(f->map + BASEADDR_RTDS); @@ -397,8 +482,7 @@ retry: len = fifo_read(&f->fifo, (char *) data_rx, 0x1000, irq_fifo); } /* Restore IER */ - XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier); - + XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier); +#endif return 0; } -#endif \ No newline at end of file diff --git a/src/fpga.c b/src/fpga.c index db1fbbbe7..3fce8db69 100644 --- a/src/fpga.c +++ b/src/fpga.c @@ -9,10 +9,8 @@ #include #include #include -#include +#include #include -#include -#include #include #include @@ -26,59 +24,59 @@ #include "config.h" #include "config-fpga.h" -#include "fpga-tests.h" -#include "fpga-bench.h" - -static struct pci_access *pacc; +/* Declarations */ +int fpga_bench(struct vfpga *f); +int fpga_tests(struct vfpga *f); void usage(char *name) { - printf("Usage: %s [ARGS]\n", name); - printf(" -d Set log level\n"); - printf(" -i ID pair of PCI device: [vendor]:[device][:class]\n"); - printf(" -s Slot of PCI device: [[[domain]:][bus]:][slot][.[func]]\n\n"); - printf("fpga PCIutil %s (built on %s %s)\n", - VERSION, __DATE__, __TIME__); - printf(" copyright 2014-2015, Institute for Automation of Complex Power Systems, EONERC\n"); - printf(" Steffen Vogel \n"); + printf("Usage: %s CONFIGFILE CMD [OPTIONS]\n", name); + printf(" Commands:\n"); + printf(" tests Test functionality of VILLASfpga card\n"); + printf(" bench Do benchmarks\n\n"); + printf(" Options:\n"); + printf(" -d Set log level\n\n"); + + print_copyright(); + + exit(EXIT_FAILURE); } int main(int argc, char *argv[]) { int ret; - char *err; - - struct vfpga fpga; - struct vfio_container vc; - struct pci_dev *pdev; - struct pci_filter filter; - - /* Init libpci */ - pacc = pci_alloc(); /* Get the pci_access structure */ - pci_init(pacc); /* Initialize the PCI library */ - pci_scan_bus(pacc); /* We want to get the list of devices */ - pci_filter_init(pacc, &filter); - - /* Default filter */ - filter.vendor = PCI_VID_XILINX; - filter.device = PCI_PID_VFPGA; + struct vfpga *fpga; + enum { + FPGA_TESTS, + FPGA_BENCH + } subcommand; + config_t config; + if (argc < 3) + usage(argv[0]); + + if (strcmp(argv[2], "tests") == 0) + subcommand = FPGA_TESTS; + else if (strcmp(argv[2], "bench") == 0) + subcommand = FPGA_BENCH; + else + usage(argv[0]); + + /* Setup libconfig */ + config_init(&config); + ret = config_read_file(&config, argv[1]); + if (ret != CONFIG_TRUE) { + error("Failed to parse configuration: %s in %s:%d", + config_error_text(&config), + config_error_file(&config) ? config_error_file(&config) : argv[1], + config_error_line(&config) + ); + } + /* Parse arguments */ char c, *endptr; - while ((c = getopt (argc, argv, "i:s:d:")) != -1) { + while ((c = getopt (argc-1, argv+1, "d:")) != -1) { switch (c) { - case 's': - err = pci_filter_parse_slot(&filter, optarg); - if (err) - error("Failed to parse slot: %s", err); - break; - - case 'i': - err = pci_filter_parse_id(&filter, optarg); - if (err) - error("Failed to parse ID: %s", err); - break; - case 'd': log_setlevel(strtoul(optarg, &endptr, 10), ~0); break; @@ -89,22 +87,15 @@ int main(int argc, char *argv[]) } } - /* Search for fpga card */ - pdev = pci_find_device(pacc, &filter); - if (!pdev) - error("Failed to find PCI device"); - - /* Get VFIO handles and details */ - ret = vfio_init(&vc); - if (ret) - serror("Failed to initialize VFIO"); - - /* Initialize fpga card */ - ret = vfpga_init2(&fpga, &vc, pdev); + /* Initialize VILLASfpga card */ + config_setting_t *cfg_root = config_root_setting(&config); + ret = vfpga_init(argc, argv, cfg_root); if (ret) error("Failed to initialize fpga card"); + + fpga = vfpga_get(); - vfpga_dump(&fpga); + vfpga_dump(fpga); /* Setup scheduler */ cpu_set_t set = integer_to_cpuset(AFFINITY); @@ -117,49 +108,22 @@ int main(int argc, char *argv[]) if (ret) serror("Failed to change scheduler"); - for (int i = 0; i < fpga.vd.irqs[VFIO_PCI_MSI_IRQ_INDEX].count; i++) { - ret = kernel_irq_setaffinity(fpga.vd.msi_irqs[i], AFFINITY, NULL); + for (int i = 0; i < fpga->vd.irqs[VFIO_PCI_MSI_IRQ_INDEX].count; i++) { + ret = kernel_irq_setaffinity(fpga->vd.msi_irqs[i], AFFINITY, NULL); if (ret) serror("Failed to change affinity of VFIO-MSI interrupt"); } - /* Start tests */ - ret = test_intc(&fpga); - info("INTC Test: %s", (ret == 0) ? GRN("passed") : RED("failed")); - - ret = test_xsg(&fpga, "xsg_multiply_add"); - info("XSG Test: %s", (ret == 0) ? GRN("passed") : RED("failed")); - -// ret = test_timer(&fpga); -// info("Timer Test: %s", (ret == 0) ? GRN("passed") : RED("failed")); - -// ret = test_fifo(&fpga); -// info("FIFO Test: %s", (ret == 0) ? GRN("passed") : RED("failed")); - - ret = test_dma(&fpga); - info("DMA Test: %s", (ret == 0) ? GRN("passed") : RED("failed")); - -// ret = test_rtds_rtt(&fpga); -// info("RTDS RTT Test: %s", (ret == 0) ? GRN("passed") : RED("failed")); - -// ret = test_rtds_cbuilder(&fpga); -// info("RTDS RTT Test: %s", (ret == 0) ? GRN("passed") : RED("failed")); - - /* Start benchmarks */ -#if 0 - ret = bench_memcpy(&fpga); -#endif + /* Start subcommand */ + switch (subcommand) { + case FPGA_TESTS: fpga_tests(fpga); break; + case FPGA_BENCH: /*fpga_bench(fpga);*/ break; + } /* Shutdown */ - ret = vfpga_deinit2(&fpga); + ret = vfpga_deinit(&fpga); if (ret) error("Failed to de-initialize fpga card"); - ret = vfio_destroy(&vc); - if (ret) - error("Failed to deinitialize VFIO module"); - - pci_cleanup(pacc); - return 0; }