Mirrors/VILLASnode
1
0
Fork 0
mirror of https://git.rwth-aachen.de/acs/public/villas/node/ synced 2025-03-09 00:00:00 +01:00
Code Issues Releases Wiki Activity

first steps towards flexible and configurable VILLASfpga / VILLASnode integration

Browse source
This commit is contained in:
Steffen Vogel 2016-06-15 20:05:09 +02:00
parent bffb47dca8
commit 98fb370e85
15 changed files with 709 additions and 846 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
Makefile
View file

@ -64,7 +64,8 @@ endif
# Enable VILLASfpga support when libpci is available
ifeq ($(shell pkg-config libpci; echo $$?),0)
LIB_OBJS += vfpga.o pci.o dma.o model.o fifo.o xsg.o vfio.o switch.o rtds_axis.o
LIB_OBJS += vfpga.o pci.o ip.o vfio.o
LIB_OBJS += dma.o model.o fifo.o switch.o rtds_axis.o
LDLIBS += -lxil
LDFLAGS += -Lthirdparty/xilinx -Wl,-rpath-link,'$$ORIGIN/thirdparty/xilinx'
CFLAGS += -Ithirdparty/xilinx/include

77
config-fpga.h
View file

@ -28,83 +28,6 @@
#define virt_to_dma(virt, dma) ((char *) ((char *) virt - dma) + BASEADDR_HOST)
#define dma_to_virt(addr, dma) ((char *) ((char *) addr - BASEADDR_HOST) + dma)
#if 0
/** Base addresses in AXI bus of FPGA */
enum {
BASEADDR_BRAM = 0x0000, /* 8 KB Block RAM for DMA SG descriptors */
BASEADDR_XSG = 0x2000, /* Xilinx System Generator Model */
BASEADDR_DMA_SG = 0x3000, /* Scatter Gather DMA controller registers */
BASEADDR_TIMER = 0x4000, /* Timer Counter registers */
BASEADDR_SWITCH = 0x5000, /* AXI Stream switch registers */
BASEADDR_FIFO_MM = 0x6000, /* Memory-mapped to AXI Stream FIFO */
BASEADDR_RESET = 0x7000, /* Internal Reset register */
BASEADDR_RTDS = 0x8000, /* RTDS to AXI Stream bridge */
BASEADDR_HLS = 0x9000, /* High-level Synthesis model registers */
BASEADDR_DMA_SIMPLE = 0xA000, /* Simple DMA controller registers */
BASEADDR_INTC = 0xB000, /* PCIe MSI Interrupt controller registers */
BASEADDR_FIFO_MM_AXI4 = 0xC000, /* AXI4 (full) interface of AXI Stream FIFO */
BASEADDR_HOST = 0x80000000 /* Start of host memory (0x0 of IOMMU) */
};
/** MSI vector table for VILLASfpga components */
enum {
MSI_TMRCTR0 = 0, /* Sensivity: rising edge, synchronous: axi_clk */
MSI_TMRCTR1 = 1, /* Sensivity: rising edge, synchronous: axi_clk */
MSI_FIFO_MM = 2, /* Sensivity: level high, synchronous: axi_clk */
MSI_DMA_MM2S = 3, /* Sensivity: level high, synchronous: axi_clk */
MSI_DMA_S2MM = 4, /* Sensivity: level high, synchronous: axi_clk */
MSI_RTDS_TS = 5, /* Sensivity: rising edge, synchronous: rtds_clk100m (New timestep started) */
MSI_RTDS_OVF = 6, /* Sensivity: rising edge, synchronous: rtds_clk100m (UserTxInProgress overlapped with UserTStepPulse)*/
MSI_RTDS_CASE = 7 /* Sensivity: rising edge, synchronous: rtds_clk100m (New RSCAD case started) */
};
/** Mapping of VILLASfpga components to AXI Stream interconnect ports */
enum {
AXIS_SWITCH_RTDS = 0,
AXIS_SWITCH_DMA_SG = 1,
AXIS_SWITCH_FIFO_MM = 2,
AXIS_SWITCH_FIFO0 = 3,
AXIS_SWITCH_XSG = 4,
AXIS_SWITCH_HLS = 5,
AXIS_SWITCH_FIFO1 = 6,
AXIS_SWITCH_DMA_SIMPLE = 7,
AXI_SWITCH_NUM = 8 /* Number of master and slave interfaces */
};
#else
#define HAS_DMA_SIMPLE 1
#define HAS_SWITCH 1
#define HAS_XSG 1
/** Base addresses in AXI bus of FPGA */
enum {
BASEADDR_SWITCH = 0x0000, /* AXI Stream switch registers */
BASEADDR_DMA_SIMPLE = 0x1000, /* Simple DMA controller registers */
BASEADDR_RESET = 0x2000, /* Internal Reset register */
BASEADDR_RTDS = 0x3000, /* RTDS to AXI Stream bridge */
BASEADDR_XSG = 0x4000, /* Xilinx System Generator Model */
BASEADDR_INTC = 0x5000, /* PCIe MSI Interrupt controller registers */
BASEADDR_PCIE = 0x10000000, /* PCIe config space */
BASEADDR_HOST = 0x80000000 /* Start of host memory (0x0 of IOMMU) */
};
/** MSI vector table for VILLASfpga components */
enum {
MSI_DMA_MM2S = 0, /* Sensivity: level high, synchronous: axi_clk */
MSI_DMA_S2MM = 1, /* Sensivity: level high, synchronous: axi_clk */
MSI_RTDS_TS = 2, /* Sensivity: rising edge, synchronous: rtds_clk100m (New timestep started) */
MSI_RTDS_OVF = 3, /* Sensivity: rising edge, synchronous: rtds_clk100m (UserTxInProgress overlapped with UserTStepPulse)*/
MSI_RTDS_CASE = 4 /* Sensivity: rising edge, synchronous: rtds_clk100m (New RSCAD case started) */
};
/** Mapping of VILLASfpga components to AXI Stream interconnect ports */
enum {
AXIS_SWITCH_RTDS = 0,
AXIS_SWITCH_XSG = 1,
AXIS_SWITCH_DMA_SIMPLE = 2,
AXI_SWITCH_NUM = 3 /* Number of master and slave interfaces */
};
#endif
#define PCI_VID_XILINX 0x10ee
#define PCI_PID_VFPGA 0x7022

40
etc/vfpga.conf Normal file
View file

@ -0,0 +1,40 @@
fpga = {
reset = false;
ips = {
switch_0 = {
vlnv = "xilinx.com:ip:axis_interconnect:2.1"
baseaddr = 0x0000;
numports = 3;
},
rtds_0 = {
vlnv = "acs.eonerc.rwth-aachen.de:user:rtds_axis:1.0"
baseaddr = 0x3000;
port = 0;
},
dma_0 = {
vlnv = "xilinx.com:ip:axi_dma:7.1";
baseaddr = 0x0000;
port = 2;
irq = 0
},
hls_dft_0 = {
vlnv = "acs.eonerc.rwth-aachen.de:hls:hls_dft:1.0";
baseaddr = 0x0000;
port = 1;
switch = "switch_0";
}
}
/* Configure switch */
paths = (
{ in = "dma", out = "hls_dft" },
{ in = "hls_dft", out = "dma" }
)
}
nodes = {
fpga_dma = {
}
}

21
include/fpga-bench.h
View file

@ -1,21 +0,0 @@
/** Benchmarks for VILLASfpga
*
* @file
* @author Steffen Vogel <stvogel@eonerc.rwth-aachen.de>
* @copyright 2015-2016, Steffen Vogel
* This file is part of VILLASnode. All Rights Reserved. Proprietary and confidential.
* Unauthorized copying of this file, via any medium is strictly prohibited.
**********************************************************************************/
#ifndef _BENCH_H_
#define _BENCH_H_
#include "nodes/vfpga.h"
int bench_memcpy(struct vfpga *f);
int bench_memcpy(struct vfpga *f);
int bench_dm(struct vfpga *f);
#endif /* _BENCH_H_ */

29
include/fpga-tests.h
View file

@ -1,29 +0,0 @@
/** Test procedures for VILLASfpga
*
* @file
* @author Steffen Vogel <stvogel@eonerc.rwth-aachen.de>
* @copyright 2016, Steffen Vogel
* This file is part of VILLASnode. All Rights Reserved. Proprietary and confidential.
* Unauthorized copying of this file, via any medium is strictly prohibited.
*********************************************************************************/
#ifndef _TESTS_H_
#define _TESTS_H_
#include "nodes/vfpga.h"
int test_intc(struct vfpga *f);
int test_xsg(struct vfpga *f, const char *name);
int test_fifo(struct vfpga *f);
int test_dma(struct vfpga *f);
int test_timer(struct vfpga *f);
int test_rtds_rtt(struct vfpga *f);
int test_rtds_cbuilder(struct vfpga *f);
#endif /* _TESTS_H_ */

63
include/villas/fpga/model.h
View file

@ -13,27 +13,19 @@
#include <stdlib.h>
#include <inttypes.h>
#include <villas/list.h>
#include "list.h"
#include "xsg.h"
#define XSG_MAGIC 0xDEADBABE
enum model_type {
MODEL_TYPE_HLS,
MODEL_TYPE_XSG
};
struct model {
char *name; /**< Name / Identifier of model */
void *baseaddr; /**< Base of register map. */
enum model_type type; /**< Model type. */
struct list parameters; /**< List of model parameters. */
union {
struct xsg_model xsg; /**< XSG specific model data */
//struct hls_model hls; /**< HLS specific model data */
};
enum model_xsg_block_type {
XSG_BLOCK_GATEWAY_IN = 0x1000,
XSG_BLOCK_GATEWAY_OUT = 0x1001,
XSG_BLOCK_INFO = 0x2000
};
enum model_param_type {
@ -56,6 +48,31 @@ union model_param_value {
bool bol;
};
struct model {
char *xml; /**< Path to a XML file which describes this model */
enum model_type type;
struct list parameters; /**< List of model parameters. */
struct list infos;
char *map;
size_t maplen;
union {
struct xsg_model {
char *map;
size_t maplen;
} xsg; /**< XSG specific model data */
struct hls_model {
} hls; /**< HLS specific model data */
};
};
struct model_info {
char *field;
char *value;
};
struct model_param {
char *name; /**< Name of the parameter */
@ -63,25 +80,25 @@ struct model_param {
enum model_param_type type; /**< Data type. Integers are represented by MODEL_GW_TYPE_(U)FIX with model_gw::binpt == 0 */
int binpt; /**< Binary point for type == MODEL_GW_TYPE_(U)FIX */
int offset; /**< Register offset to model::baseaddress */
uintptr_t offset; /**< Register offset to model::baseaddress */
union model_param_value default_value;
struct model *model; /**< A pointer to the model structure to which this parameters belongs to. */
};
int model_init(struct model *m, const char *xml);
int model_init(struct model *m, uintptr_t baseaddr);
int model_init_from_xsg_map(struct model *m, uintptr_t baseaddr);
int model_init_from_xml(struct model *m);
void model_destroy(struct model *m);
void model_dump(struct model *m);
void model_param_destroy(struct model_param *r);
void model_param_add(struct model *m, const char *name, enum model_param_direction dir, enum model_param_type type);
char * xsg_get_info(struct xsg_model *x, const char *key);
/** Read a model parameter.
*
* Note: the data type of the register is taken into account.
@ -97,4 +114,10 @@ int model_param_read(struct model_param *p, double *v);
*/
int model_param_write(struct model_param *p, double v);
/** Read the XSG model information from ROM */
int model_xsg_map_read(void *offset, void *dst, size_t len);
/** Parse binary model information from ROM */
int model_xsg_map_parse(uint32_t *map, size_t len, struct list *parameters, struct list *infos);
#endif /* _MODEL_H_ */

54
include/villas/fpga/xsg.h
View file

@ -1,54 +0,0 @@
/** Interface to Xilinx System Generator Models via VILLASfpga
*
* @file
* @author Steffen Vogel <stvogel@eonerc.rwth-aachen.de>
* @copyright 2016, Steffen Vogel
* This file is part of S2SS. All Rights Reserved. Proprietary and confidential.
* Unauthorized copying of this file, via any medium is strictly prohibited.
*********************************************************************************/
#ifndef _XSG_H_
#define _XSG_H_
#define XSG_MAGIC 0xDEADBABE
/* Forward declaration */
struct model;
enum xsg_block_type {
XSG_BLOCK_GATEWAY_IN = 0x1000,
XSG_BLOCK_GATEWAY_OUT = 0x1001,
XSG_BLOCK_INFO = 0x2000
};
struct xsg_model {
int version;
char *map;
size_t maplen;
struct list infos;
};
struct xsg_info {
char *field;
char *value;
};
int xsg_init_from_map(struct model *m);
int xsg_init_from_xml(struct model *m, const char *xml);
void xsg_destroy(struct xsg_model *x);
void xsg_dump(struct xsg_model *x);
void xsg_info_destroy(struct xsg_info *i);
/** Read the XSG model information from ROM */
int xsg_map_read(void *offset, void *dst, size_t len);
/** Parse binary model information from ROM */
int xsg_map_parse(uint32_t *map, size_t len, struct list *parameters, struct list *infos);
#endif /* _XSG_H_ */

54
include/villas/nodes/vfpga.h
View file

@ -17,53 +17,36 @@
#ifndef _VFPGA_H_
#define _VFPGA_H_
#include <xilinx/xtmrctr.h>
#include <xilinx/xintc.h>
#include <xilinx/xllfifo.h>
#include <xilinx/xaxis_switch.h>
#include <xilinx/xaxidma.h>
#include "kernel/vfio.h"
#include "fpga/switch.h"
#include "fpga/model.h"
#include "fpga/dma.h"
#include "fpga/fifo.h"
#include "fpga/rtds_axis.h"
#include <pci/pci.h>
#include "nodes/vfpga.h"
#include "node.h"
#include "list.h"
#define BASEADDR_HOST 0x8000000
#define VFPGA_BAR 0 /**< The Base Address Register which is mmap()ed to the User Space */
struct vfpga {
struct pci_filter filter;
enum {
vfpga_MODE_FIFO,
vfpga_MODE_DMA
} mode;
int features;
struct list models; /**< List of XSG and HLS model blocks on FPGA. */
struct pci_filter filter; /**< Filter for libpci with device id & slot */
struct vfio_dev vd; /**< VFIO device handle. */
/* Xilinx IP blocks */
XTmrCtr tmr;
XLlFifo fifo;
XAxiDma dma_simple;
XAxiDma dma_sg;
XAxis_Switch sw;
int reset; /**< Reset VILLASfpga during startup? */
struct list ips; /**< List of IP components on FPGA. */
char *map; /**< PCI BAR0 mapping for register access */
char *dma; /**< DMA mapped memory */
size_t maplen;
size_t dmalen;
/** Base addresses for internal IP blocks */
struct {
uintptr_t reset;
uintptr_t intc;
} baseaddr;
};
/** @see node_vtable::init */
@ -73,7 +56,9 @@ int vfpga_init(int argc, char * argv[], config_setting_t *cfg);
int vfpga_deinit();
/** @see node_vtable::parse */
int vfpga_parse(struct node *n, config_setting_t *cfg);
int vfpga_parse_node(struct node *n, config_setting_t *cfg);
int vfpga_parse(struct vfpga *v, int argc, char * argv[], config_setting_t *cfg);
/** @see node_vtable::print */
char * vfpga_print(struct node *n);
@ -90,14 +75,13 @@ int vfpga_read(struct node *n, struct sample *smps[], unsigned cnt);
/** @see node_vtable::write */
int vfpga_write(struct node *n, struct sample *smps[], unsigned cnt);
//////////
int vfpga_init2(struct vfpga *f, struct vfio_container *vc, struct pci_dev *pdev);
int vfpga_deinit2(struct vfpga *f);
/** Get pointer to internal VILLASfpga datastructure */
struct vfpga * vfpga_get();
/** Reset VILLASfpga */
int vfpga_reset(struct vfpga *f);
/** Dump some details about the fpga card */
void vfpga_dump(struct vfpga *f);
#endif /** _VFPGA_H_ @} */

12
lib/cfg.c
View file

@ -20,16 +20,18 @@
int config_parse(const char *filename, config_t *cfg, struct settings *set,
struct list *nodes, struct list *paths)
{
char *filename_cpy = strdup(filename);
char *include_dir = dirname(filename_cpy);
int ret;
char *filename_cpy, include_dir;
filename_cpy = strdup(filename);
include_dir = dirname(filename_cpy);
free(filename_cpy);
/* Setup libconfig */
config_set_auto_convert(cfg, 1);
config_set_include_dir(cfg, include_dir);
free(filename_cpy);
int ret = strcmp("-", filename) ? config_read_file(cfg, filename) : config_read(cfg, stdin);
ret = strcmp("-", filename) ? config_read_file(cfg, filename) : config_read(cfg, stdin);
if (ret != CONFIG_TRUE) {
error("Failed to parse configuration: %s in %s:%d",
config_error_text(cfg),

161
lib/fpga/model.c
View file

@ -15,20 +15,40 @@
#include <villas/log.h>
#include "fpga/model.h"
#include "fpga/xsg.h"
int model_init(struct model *m, const char *xml)
int model_init(struct model *m, uintptr_t baseaddr)
{
/** @todo: parse XML model descriptions */
int ret;
/** @todo: Currently only XSG models are supported */
list_init(&m->parameters);
list_init(&m->infos);
ret = model_init_from_xsg_map(m, baseaddr);
if (ret)
error("Failed to init XSG model: %d", ret);
return 0;
}
static void model_param_destroy(struct model_param *p)
{
free(p->name);
}
static void model_info_destroy(struct model_info *i)
{
free(i->field);
free(i->value);
}
void model_destroy(struct model *m)
{
list_destroy(&m->parameters, (dtor_cb_t) model_param_destroy, true);
list_destroy(&m->infos, (dtor_cb_t) model_info_destroy, true);
free(m->map);
}
void model_dump(struct model *m)
@ -37,13 +57,13 @@ void model_dump(struct model *m)
const char *model_types[] = { "HLS", "XSG" };
const char *parameter_dirs[] = { "In", "Out", "In/Out" };
info("Model: %s (baseaddr=%p, type=%s)", m->name, m->baseaddr, model_types[m->type]);
info("Model: (type=%s)", model_types[m->type]);
{ INDENT
info("Parameters:");
list_foreach(struct model_param *p, &m->parameters) { INDENT
if (p->direction == MODEL_PARAM_IN)
info("%#x: %s (%s) = %.3f %s %u",
info("%#jx: %s (%s) = %.3f %s %u",
p->offset,
p->name,
parameter_dirs[p->direction],
@ -52,27 +72,23 @@ void model_dump(struct model *m)
p->binpt
);
else if (p->direction == MODEL_PARAM_OUT)
info("%#x: %s (%s)",
info("%#jx: %s (%s)",
p->offset,
p->name,
parameter_dirs[p->direction]
);
}
switch (m->type) {
case MODEL_TYPE_HLS:
//hls_dump(&m->hls);
break;
case MODEL_TYPE_XSG:
xsg_dump(&m->xsg);
break;
info("Infos:");
list_foreach(struct model_info *i, &m->infos) { INDENT
info("%s: %s", i->field, i->value);
}
}
}
int model_param_read(struct model_param *p, double *v)
{
union model_param_value *ptr = p->offset + p->model->baseaddr;
union model_param_value *ptr = p->offset;// TODO: + p->model->baseaddr;
switch (p->type) {
case MODEL_PARAM_TYPE_UFIX:
@ -96,7 +112,7 @@ int model_param_read(struct model_param *p, double *v)
int model_param_write(struct model_param *p, double v)
{
union model_param_value *ptr = p->offset + p->model->baseaddr;
union model_param_value *ptr = p->offset;// TODO: + p->model->baseaddr;
switch (p->type) {
case MODEL_PARAM_TYPE_UFIX:
@ -130,7 +146,118 @@ void model_param_add(struct model *m, const char *name, enum model_param_directi
list_push(&m->parameters, p);
}
void model_param_destroy(struct model_param *p)
int model_init_from_xsg_map(struct model *m, uintptr_t baseaddr)
{
free(p->name);
int ret;
list_init(&m->infos);
m->map = alloc(1024);
m->maplen = model_xsg_map_read(baseaddr, m->map, 1024 / 4);
if (m->maplen < 0)
return -1;
debug(5, "XSG: memory map length = %#zx", m->maplen);
ret = model_xsg_map_parse((uint32_t *) m->map, m->maplen, &m->parameters, &m->infos);
if (ret)
return -2;
return 0;
}
static uint32_t model_xsg_map_checksum(uint32_t *map, size_t len)
{
uint32_t chks = 0;
for (int i = 2; i < len-1; i++)
chks += map[i];
return chks; /* moduluo 2^32 because of overflow */
}
int model_xsg_map_parse(uint32_t *map, size_t len, struct list *parameters, struct list *infos)
{
#define copy_string(off) strndup((char *) (data + (off)), (length - (off)) * 4);
struct model_param *p;
struct model_info *i;
int j;
if (map[0] != XSG_MAGIC)
error("Invalid magic: %#x", map[0]);
for (j = 2; j < len-1;) {
uint16_t type = map[j] & 0xFFFF;
uint16_t length = map[j] >> 16;
uint32_t *data = &map[j+1];
switch (type) {
case XSG_BLOCK_GATEWAY_IN:
case XSG_BLOCK_GATEWAY_OUT:
if (length < 4)
break; /* block is to small to describe a gateway */
p = alloc(sizeof(*p));
p->name = copy_string(3);
p->default_value.flt = *((float *) &data[1]);
p->offset = data[2];
p->direction = type & 0x1;
p->type = (data[0] >> 0) & 0xFF;
p->binpt = (data[0] >> 8) & 0xFF;
list_push(parameters, p);
break;
case XSG_BLOCK_INFO:
i = alloc(sizeof(struct model_info));
i->field = copy_string(0);
i->value = copy_string((int) ceil((double) (strlen(i->field) + 1) / 4))
list_push(infos, i);
break;
default:
warn("Unknown block type: %#06x", type);
}
j += length + 1;
}
if (model_xsg_map_checksum(map, len) != map[j])
error("XSG: Invalid checksum");
return 0;
#undef copy_string
}
int model_xsg_map_read(void *baseaddr, void *dst, size_t len)
{
#define get_word(a) ({ *addr = a; *data; })
volatile uint32_t *addr = baseaddr + 0x00;
volatile uint32_t *data = baseaddr + 0x04;
uint32_t *map = dst;
uint32_t maplen, magic;
/* Check start DW */
magic = get_word(0);
if (magic != XSG_MAGIC)
return -1;
maplen = get_word(1);
if (maplen < 3)
return -2;
/* Read Data */
int i;
for (i = 0; i < MIN(maplen, len); i++)
map[i] = get_word(i);
return i;
#undef get_word
}

176
lib/fpga/xsg.c
View file

@ -1,176 +0,0 @@
/** Interface to Xilinx System Generator Models via VILLASfpga
*
* @author Steffen Vogel <stvogel@eonerc.rwth-aachen.de>
* @copyright 2016, Steffen Vogel
* This file is part of VILLASnode. All Rights Reserved. Proprietary and confidential.
* Unauthorized copying of this file, via any medium is strictly prohibited.
*********************************************************************************/
#include <stdint.h>
#include <string.h>
#include <math.h>
#include <sys/types.h>
#include <villas/list.h>
#include <villas/utils.h>
#include <villas/log.h>
#include "fpga/model.h"
#include "fpga/xsg.h"
#include "utils.h"
int xsg_init_from_map(struct model *m)
{
int ret;
struct xsg_model *x = &m->xsg;
list_init(&x->infos);
x->map = alloc(1024);
x->maplen = xsg_map_read(m->baseaddr, x->map, 1024 / 4);
if (x->maplen < 0)
return -1;
debug(5, "XSG: memory map length = %#zx", x->maplen);
ret = xsg_map_parse((uint32_t *) x->map, x->maplen, &m->parameters, &x->infos);
if (ret)
return -2;
m->name = xsg_get_info(x, "Name");
return 0;
}
int xsg_init_from_xml(struct model *m, const char *xml)
{
/** @todo: parse Simulink .mdl file here (XML based) */
return -1;
}
void xsg_destroy(struct xsg_model *x)
{
list_destroy(&x->infos, (dtor_cb_t) xsg_info_destroy, true);
free(x->map);
}
char * xsg_get_info(struct xsg_model *x, const char *key)
{
struct xsg_info *info = list_lookup(&x->infos, key);
return info ? info->value : NULL;
}
void xsg_info_destroy(struct xsg_info *i)
{
free(i->field);
free(i->value);
}
void xsg_dump(struct xsg_model *x)
{
info("XSG Infos:");
list_foreach(struct xsg_info *i, &x->infos) { INDENT
info("%s: %s", i->field, i->value);
}
}
static uint32_t xsg_map_checksum(uint32_t *map, size_t len)
{
uint32_t chks = 0;
for (int i = 2; i < len-1; i++)
chks += map[i];
return chks; /* moduluo 2^32 because of overflow */
}
int xsg_map_parse(uint32_t *map, size_t len, struct list *parameters, struct list *infos)
{
#define copy_string(off) strndup((char *) (data + (off)), (length - (off)) * 4);
struct model_param *p;
struct xsg_info *info;
int i;
if (map[0] != XSG_MAGIC)
error("Invalid magic: %#x", map[0]);
for (i = 2; i < len-1;) {
uint16_t type = map[i] & 0xFFFF;
uint16_t length = map[i] >> 16;
uint32_t *data = &map[i+1];
switch (type) {
case XSG_BLOCK_GATEWAY_IN:
case XSG_BLOCK_GATEWAY_OUT:
if (length < 4)
break; /* block is to small to describe a gateway */
p = alloc(sizeof(*p));
p->name = copy_string(3);
p->default_value.flt = *((float *) &data[1]);
p->offset = data[2];
p->direction = type & 0x1;
p->type = (data[0] >> 0) & 0xFF;
p->binpt = (data[0] >> 8) & 0xFF;
list_push(parameters, p);
break;
case XSG_BLOCK_INFO:
info = alloc(sizeof(*info));
info->field = copy_string(0);
info->value = copy_string((int) ceil((double) (strlen(info->field) + 1) / 4))
list_push(infos, info);
break;
default:
warn("Unknown block type: %#06x", type);
}
i += length + 1;
}
if (xsg_map_checksum(map, len) != map[i])
error("XSG: Invalid checksum");
return 0;
#undef copy_string
}
int xsg_map_read(void *baseaddr, void *dst, size_t len)
{
#define get_word(a) ({ *addr = a; *data; })
volatile uint32_t *addr = baseaddr + 0x00;
volatile uint32_t *data = baseaddr + 0x04;
uint32_t *map = dst;
uint32_t maplen, magic;
/* Check start DW */
magic = get_word(0);
if (magic != XSG_MAGIC)
return -1;
maplen = get_word(1);
if (maplen < 3)
return -2;
/* Read Data */
int i;
for (i = 0; i < MIN(maplen, len); i++)
map[i] = get_word(i);
return i;
#undef get_word
}

415
lib/nodes/vfpga.c
View file

@ -7,7 +7,7 @@
*********************************************************************************/
#include <stdio.h>
#include <inttypes.h>
#include <stdint.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/mman.h>
@ -17,28 +17,202 @@
#include "nodes/vfpga.h"
#include "config.h"
#include "fpga/ip.h"
#include "config-fpga.h"
#include "utils.h"
#include "timing.h"
struct vfpga fpga;
struct vfio_container vc;
static struct pci_access *pacc;
typedef void(*log_cb_t)(char *, ...);
int vfpga_init2(struct vfpga *f, struct vfio_container *vc, struct pci_dev *pdev)
int vfpga_reset(struct vfpga *f)
{
int ret;
char state[4096];
list_init(&f->models);
/* Save current state of PCI configuration space */
ret = pread(f->vd.fd, state, sizeof(state), (off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40);
if (ret != sizeof(state))
return -1;
uint32_t *rst_reg = (uint32_t *) (f->map + f->baseaddr.reset);
info("Reset fpga");
rst_reg[0] = 1;
usleep(100000);
/* Restore previous state of PCI configuration space */
ret = pwrite(f->vd.fd, state, sizeof(state), (off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40);
if (ret != sizeof(state))
return -1;
info("Reset status = %#x", *rst_reg);
return 0;
}
void vfpga_dump(struct vfpga *f)
{
char namebuf[128];
char *name;
name = pci_lookup_name(pacc, namebuf, sizeof(namebuf), PCI_LOOKUP_DEVICE, fpga.vd.pdev->vendor_id, fpga.vd.pdev->device_id);
pci_fill_info(fpga.vd.pdev, PCI_FILL_IDENT | PCI_FILL_BASES | PCI_FILL_CLASS); /* Fill in header info we need */
info("VILLASfpga card: %s", name);
{ INDENT
info("Slot: %04x:%02x:%02x.%d", fpga.vd.pdev->domain, fpga.vd.pdev->bus, fpga.vd.pdev->dev, fpga.vd.pdev->func);
info("Vendor ID: %04x", fpga.vd.pdev->vendor_id);
info("Device ID: %04x", fpga.vd.pdev->device_id);
info("Class ID: %04x", fpga.vd.pdev->device_class);
info("BAR0 mapped at %p", fpga.map);
info("DMA mapped at %p", fpga.dma);
info("IP blocks:");
list_foreach(struct ip *i, &f->ips) { INDENT
ip_dump(i);
}
}
vfio_dump(fpga.vd.group->container);
}
struct vfpga * vfpga_get()
{
return &fpga;
}
static void vfpga_debug(char *msg, ...) {
va_list ap;
va_start(ap, msg);
log_vprint(LOG_LVL_DEBUG, msg, ap);
va_end(ap);
}
int vfpga_parse(struct vfpga *v, int argc, char * argv[], config_setting_t *cfg)
{
int ret;
const char *slot, *id, *err;
config_setting_t *cfg_fpga, *cfg_ips, *cfg_slot, *cfg_id;
/* Default values */
v->filter.vendor = PCI_VID_XILINX;
v->filter.device = PCI_PID_VFPGA;
v->reset = false;
cfg_fpga = config_setting_get_member(cfg, "fpga");
if (!cfg_fpga)
cerror(cfg, "Config file is missing VILLASfpga configuration");
config_setting_lookup_bool(cfg_fpga, "reset", &v->reset);
cfg_slot = config_setting_get_member(cfg_fpga, "slot");
if (cfg_slot) {
slot = config_setting_get_string(cfg_slot);
if (slot) {
err = pci_filter_parse_slot(&v->filter, (char*) slot);
if (err)
cerror(cfg_slot, "%s", err);
}
else
cerror(cfg_slot, "Invalid slot format");
}
cfg_id = config_setting_get_member(cfg_fpga, "id");
if (cfg_id) {
id = config_setting_get_string(cfg_id);
if (id) {
err = pci_filter_parse_id(&v->filter, (char*) id);
if (err)
cerror(cfg_id, "%s", err);
}
else
cerror(cfg_slot, "Invalid id format");
}
f->features = 0;
cfg_ips = config_setting_get_member(cfg_fpga, "ips");
if (!cfg_ips)
cerror(cfg_fpga, "FPGA configuration is missing ips section");
for (int i = 0; i < config_setting_length(cfg_ips); i++) {
struct ip ip;
config_setting_t *cfg_ip = config_setting_get_elem(cfg_ips, i);
ret = ip_parse(&ip, cfg_ip);
if (ret)
cerror(cfg_ip, "Failed to parse VILLASfpga ip");
list_push(&v->ips, memdup(&ip, sizeof(struct ip)));
}
return 0;
}
struct pci_access * vfpga_init_pci()
{
struct pci_access *pacc;
pacc = pci_alloc(); /* Get the pci_access structure */
if (!pacc)
error("Failed to allocate PCI access structure");
pci_init(pacc); /* Initialize the PCI library */
pci_scan_bus(pacc); /* We want to get the list of devices */
pacc->error = (log_cb_t) error; /* Replace logging and debug functions */
pacc->warning = (log_cb_t) warn;
pacc->debug = vfpga_debug;
pci_scan_bus(pacc); /* We want to get the list of devices */
return pacc;
}
int vfpga_init(int argc, char * argv[], config_setting_t *cfg)
{
int ret;
struct pci_access *pacc;
struct pci_dev *pdev;
struct vfpga *f;
/* For now we only support a single VILALSfpga card */
f = vfpga_get();
/* Some hardcoded addresses for internal IP blocks */
f->baseaddr.reset = 0x2000;
f->baseaddr.intc = 0x5000;
pacc = vfpga_init_pci();
pci_filter_init(pacc, &f->filter);
list_init(&f->ips);
ret = vfpga_parse(f, argc, argv, cfg);
if (ret)
cerror(cfg, "Failed to parse VILLASfpga config");
/* Search for fpga card */
pdev = pci_find_device(pacc, &f->filter);
if (!pdev)
error("Failed to find PCI device");
if (pdev->vendor_id != PCI_VID_XILINX)
error("This is not a Xilinx FPGA board!");
/* Get VFIO handles and details */
ret = vfio_init(&vc);
if (ret)
serror("Failed to initialize VFIO");
/* Attach PCIe card to VFIO container */
ret = vfio_pci_attach(&f->vd, vc, pdev);
ret = vfio_pci_attach(&f->vd, &vc, pdev);
if (ret)
error("Failed to attach VFIO device");
@ -64,223 +238,44 @@ int vfpga_init2(struct vfpga *f, struct vfio_container *vc, struct pci_dev *pdev
if (ret)
serror("Failed to enable PCI device");
#if 1
/* Trigger internal reset of fpga card / PCIe endpoint */
ret = vfpga_reset(f);
if (ret)
serror("Failed to reset fpga card");
#endif
/* Reset / detect PCI device */
ret = vfio_pci_reset(&f->vd);
if (ret)
serror("Failed to reset PCI device");
/* Initialize Interrupt controller */
XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IMR_OFFSET, 0);
XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IAR_OFFSET, 0xFFFFFFFF); /* Acknowlege all pending IRQs */
usleep(1000);
/* Setup Vectors */
for (int i = 0; i < 16; i++)
XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IVAR_OFFSET + i * sizeof(uint32_t), i);
XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IMR_OFFSET, 0xFFFFFFFF); /* Use fast acknowlegement for all IRQs */
XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, 0x00000000);
XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_MER_OFFSET, XIN_INT_HARDWARE_ENABLE_MASK | XIN_INT_MASTER_ENABLE_MASK);
#ifdef HAS_SWITCH
/* Initialize AXI4-Sream crossbar switch */
ret = switch_init(&f->sw, f->map + BASEADDR_SWITCH, AXI_SWITCH_NUM, AXI_SWITCH_NUM);
if (ret)
error("Failed to initialize switch");
#endif
#ifdef HAS_XSG
/* Initialize System Generator model */
struct model *m = alloc(sizeof(struct model));
m->baseaddr = f->map + BASEADDR_XSG;
m->type = MODEL_TYPE_XSG;
ret = model_init(m, NULL);
if (ret)
error("Failed to init model: %d", ret);
ret = xsg_init_from_map(m);
if (ret)
error("Failed to init XSG model: %d", ret);
list_push(&f->models, m);
#endif
#ifdef HAS_FIFO_MM
/* Initialize AXI4-Stream Memory Mapped FIFO */
ret = fifo_init(&f->fifo, f->map + BASEADDR_FIFO_MM, f->map + BASEADDR_FIFO_MM_AXI4);
if (ret)
error("Failed to initialize FIFO");
#endif
#ifdef HAS_DMA_SG
/* Initialize Scatter Gather DMA controller */
struct dma_mem bd = {
.base_virt = (uintptr_t) f->map + BASEADDR_BRAM,
.base_phys = BASEADDR_BRAM,
.len = 0x2000
};
struct dma_mem rx = {
.base_virt = (uintptr_t) f->dma,
.base_phys = BASEADDR_HOST,
.len = 0x10000
};
ret = dma_init(&f->dma_sg, f->map + BASEADDR_DMA_SG, DMA_MODE_SG, &bd, &rx, sizeof(uint32_t) * 64);
if (ret)
return -1;
#endif
#ifdef HAS_DMA_SIMPLE
/* Initialize Simple DMA controller */
ret = dma_init(&f->dma_simple, f->map + BASEADDR_DMA_SIMPLE, DMA_MODE_SIMPLE, 0, 0, 0);
if (ret)
return -1;
#endif
#ifdef HAS_TIMER
/* Initialize timer */
XTmrCtr_Config tmr_cfg = {
.BaseAddress = (uintptr_t) f->map + BASEADDR_TIMER,
.SysClockFreqHz = AXI_HZ
};
XTmrCtr_CfgInitialize(&f->tmr, &tmr_cfg, (uintptr_t) f->map + BASEADDR_TIMER);
XTmrCtr_InitHw(&f->tmr);
#endif
return 0;
}
int vfpga_deinit2(struct vfpga *f)
{
list_destroy(&f->models, (dtor_cb_t) model_destroy, true);
return 0;
}
int vfpga_reset(struct vfpga *f)
{
int ret;
char state[4096];
/* Save current state of PCI configuration space */
ret = pread(f->vd.fd, state, sizeof(state), (off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40);
if (ret != sizeof(state))
return -1;
uint32_t *rst_reg = (uint32_t *) (f->map + BASEADDR_RESET);
info("Reset fpga");
rst_reg[0] = 1;
usleep(100000);
/* Restore previous state of PCI configuration space */
ret = pwrite(f->vd.fd, state, sizeof(state), (off_t) VFIO_PCI_CONFIG_REGION_INDEX << 40);
if (ret != sizeof(state))
return -1;
info("Reset status = %#x", *rst_reg);
return 0;
}
/* Dump some details about the fpga card */
void vfpga_dump(struct vfpga *f)
{
char namebuf[128];
char *name;
name = pci_lookup_name(pacc, namebuf, sizeof(namebuf), PCI_LOOKUP_DEVICE, f->vd.pdev->vendor_id, f->vd.pdev->device_id);
pci_fill_info(f->vd.pdev, PCI_FILL_IDENT | PCI_FILL_BASES | PCI_FILL_CLASS); /* Fill in header info we need */
info("Found fpga card: %s", name);
{ INDENT
info("slot=%04x:%02x:%02x.%d", f->vd.pdev->domain, f->vd.pdev->bus, f->vd.pdev->dev, f->vd.pdev->func);
info("vendor=%04x", f->vd.pdev->vendor_id);
info("device=%04x", f->vd.pdev->device_id);
info("class=%04x", f->vd.pdev->device_class);
if (f->reset) {
ret = vfpga_reset(f);
if (ret)
serror("Failed to reset fpga card");
/* Reset / detect PCI device */
ret = vfio_pci_reset(&f->vd);
if (ret)
serror("Failed to reset PCI device");
}
info("BAR0 mapped at %p", f->map);
vfio_dump(f->vd.group->container);
}
static void vfpga_debug(char *msg, ...) {
va_list ap;
va_start(ap, msg);
log_vprint(LOG_LVL_DEBUG, msg, ap);
va_end(ap);
}
int vfpga_init(int argc, char * argv[], config_setting_t *cfg)
{
if (getuid() != 0)
error("The vfpga node-type requires superuser privileges!");
pacc = pci_alloc(); /* Get the pci_access structure */
if (!pacc)
error("Failed to allocate PCI access structure");
pci_init(pacc); /* Initialize the PCI library */
pacc->error = (log_cb_t) error; /* Replace logging and debug functions */
pacc->warning = (log_cb_t) warn;
pacc->debug = vfpga_debug;
pci_scan_bus(pacc); /* We want to get the list of devices */
list_foreach(struct ip *c, &f->ips) {
c->card = f;
ip_init(c);
}
return 0;
}
int vfpga_deinit()
{
int ret;
list_destroy(&fpga.ips, (dtor_cb_t) ip_destroy, true);
pci_cleanup(pacc);
ret = vfio_destroy(&vc);
if (ret)
error("Failed to deinitialize VFIO module");
return 0;
}
int vfpga_parse(struct node *n, config_setting_t *cfg)
int vfpga_parse_node(struct node *n, config_setting_t *cfg)
{
struct vfpga *v = n->_vd;
const char *slot, *id, *err;
config_setting_t *cfg_slot, *cfg_id;
pci_filter_init(NULL, &v->filter);
cfg_slot = config_setting_get_member(cfg, "slot");
if (cfg_slot) {
slot = config_setting_get_string(cfg_slot);
if (slot) {
err = pci_filter_parse_slot(&v->filter, (char*) slot);
if (err)
cerror(cfg_slot, "%s", err);
}
else
cerror(cfg_slot, "Invalid slot format");
}
cfg_id = config_setting_get_member(cfg, "id");
if (cfg_id) {
id = config_setting_get_string(cfg_id);
if (id) {
err = pci_filter_parse_id(&v->filter, (char*) id);
if (err)
cerror(cfg_id, "%s", err);
}
else
cerror(cfg_slot, "Invalid id format");
}
// struct vfpga *v = n->_vd;
return 0;
}
@ -339,8 +334,8 @@ int vfpga_write(struct node *n, struct sample *smps[], unsigned cnt)
static struct node_type vt = {
.name = "vfpga",
.description = "VILLASfpga PCIe card (libpci)",
.vectorize = 0,
.parse = vfpga_parse,
.vectorize = 1,
.parse = vfpga_parse_node,
.print = vfpga_print,
.open = vfpga_open,
.close = vfpga_close,

6
src/fpga-bench.c
View file

@ -31,7 +31,7 @@ enum benchmarks {
MAX_BENCHS
};
static int bench_run(struct vfpga *f, int polling, int bench)
int fpga_bench(struct vfpga *f, int polling, int bench)
{
struct hist hist;
uint64_t start, stop;
@ -87,7 +87,7 @@ static int bench_run(struct vfpga *f, int polling, int bench)
hist_destroy(&hist);
}
int bench_dm(struct vfpga *f)
int fpga_bench_dm(struct vfpga *f)
{
int irq_fifo, irq_dma_mm2s, irq_dma_s2mm;
int polling = true;
@ -106,7 +106,7 @@ int bench_dm(struct vfpga *f)
return 0;
}
int bench_memcpy(struct vfpga *f)
int fpga_bench_memcpy(struct vfpga *f)
{
uint64_t start, end, total = 0;

296
src/fpga-tests.c
View file

@ -13,10 +13,19 @@
#include <poll.h>
#include <time.h>
#include <xilinx/xtmrctr.h>
#include <xilinx/xintc.h>
#include <xilinx/xllfifo.h>
#include <xilinx/xaxis_switch.h>
#include <xilinx/xaxidma.h>
#include <villas/utils.h>
#include <villas/nodes/vfpga.h>
#include "fpga-tests.h"
#include <villas/fpga/ip.h>
//#include <villas/fpga/cbmodel.h>
#include "config.h"
#include "config-fpga.h"
@ -26,83 +35,140 @@
#define CPU_HZ 3392389000
int test_intc(struct vfpga *f)
/* Declarations */
int fpga_test_intc(struct vfpga *f);
int fpga_test_timer(struct vfpga *f);
int fpga_test_fifo(struct vfpga *f);
int fpga_test_dma(struct vfpga *f);
int fpga_test_xsg(struct vfpga *f);
int fpga_test_rtds_rtt(struct vfpga *f);
int fpga_test_rtds_cbuilder(struct vfpga *f);
int fpga_tests(struct vfpga *f)
{
int ret;
struct {
const char *name;
int (*func)(struct vfpga *f);
} tests[] = {
{ "Interrupt Controller", fpga_test_intc },
{ "Timer Counter", fpga_test_timer },
{ "FIFO", fpga_test_fifo },
{ "DMA", fpga_test_dma },
{ "XSG: multiply_add", fpga_test_xsg },
{ "RTDS: RTT Test", fpga_test_rtds_rtt },
{ "RTDS: CBuilder", fpga_test_rtds_cbuilder }
};
for (int i = 0; i < ARRAY_LEN(tests); i++) {
ret = tests[i].func(f);
info("%s: %s", tests[i].name, (ret == 0) ? GRN("passed") : RED("failed"));
}
return 0;
}
int fpga_test_intc(struct vfpga *f)
{
uint32_t ier, isr;
/* Save old IER */
ier = XIntc_In32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET);
XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier | 0xFF00);
ier = XIntc_In32(f->baseaddr.intc + XIN_IER_OFFSET);
XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier | 0xFF00);
XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_ISR_OFFSET, 0xFF00);
XIntc_Out32(f->baseaddr.intc + XIN_ISR_OFFSET, 0xFF00);
debug(3, "Wait for 8 SW triggerd interrupts");
for (int i = 0; i < 8; i++)
wait_irq(f->vd.msi_efds[i+8]);
isr = XIntc_In32((uintptr_t) f->map + BASEADDR_INTC + XIN_ISR_OFFSET);
isr = XIntc_In32(f->baseaddr.intc + XIN_ISR_OFFSET);
/* Restore IER */
XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier);
XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier);
return (isr & 0xFF00) ? -1 : 0; /* ISR should get cleared by MSI_Grant_signal */
}
#ifdef HAS_XSG
int test_xsg(struct vfpga *f, const char *name)
int fpga_test_xsg(struct vfpga *f)
{
struct model *m;
struct ip *xsg, *dma, *sw;
xsg = ip_vlnv_lookup(&f->ips, NULL, "xsg", "xsg_multiply_add", NULL);
sw = ip_vlnv_lookup(&f->ips, "xilinx.com", "ip", "axis_interconnect", NULL);
dma = ip_vlnv_lookup(&f->ips, "xilinx.com", "ip", "axi_dma", NULL);
m = list_lookup(&f->models, name);
if (m) {
model_dump(m);
/* Check if required IP is available on FPGA */
if (!dma || !xsg || !dma)
return -1;
list_foreach(struct model_param *p, &m->parameters) {
if (p->direction == MODEL_PARAM_IN) {
model_param_write(p, p->default_value.flt);
info("Param '%s' updated to: %f", p->name, p->default_value.flt);
}
}
ip_dump(xsg);
if (strcmp(m->name, "xsg_multiply_add") == 0) {
switch_connect(&f->sw, AXIS_SWITCH_DMA_SIMPLE, AXIS_SWITCH_XSG);
switch_connect(&f->sw, AXIS_SWITCH_XSG, AXIS_SWITCH_DMA_SIMPLE);
float *src = (float *) f->dma;
float *dst = (float *) f->dma + TEST_LEN;
for (int i = 0; i < 6; i++)
src[i] = 1.1 * (i+1);
ssize_t len = dma_ping_pong(&f->dma_simple, virt_to_dma(src, f->dma), virt_to_dma(dst, f->dma), 6 * sizeof(float), -1);
if (len != 6 * sizeof(float))
error("Failed to to ping pong DMA transfer: %zd", len);
for (int i = 0; i < 6; i++)
info("src[%d] = %f", i, src[i]);
for (int i = 0; i < 6; i++)
info("dst[%d] = %f", i, dst[i]);
list_foreach(struct model_param *p, &xsg->model.parameters) {
if (p->direction == MODEL_PARAM_IN) {
model_param_write(p, p->default_value.flt);
info("Param '%s' updated to: %f", p->name, p->default_value.flt);
}
}
else
warn("There is no model named '%s'", name);
switch_connect(&sw->sw, dma->port, xsg->port);
switch_connect(&sw->sw, xsg->port, dma->port);
float *src = (float *) f->dma;
float *dst = (float *) f->dma + TEST_LEN;
for (int i = 0; i < 6; i++)
src[i] = 1.1 * (i+1);
ssize_t len = dma_ping_pong(&dma->dma, virt_to_dma(src, f->dma), virt_to_dma(dst, f->dma), 6 * sizeof(float), -1);
if (len != 6 * sizeof(float))
error("Failed to to ping pong DMA transfer: %zd", len);
for (int i = 0; i < 6; i++)
info("src[%d] = %f", i, src[i]);
for (int i = 0; i < 6; i++)
info("dst[%d] = %f", i, dst[i]);
return 0;
}
#endif
#ifdef HAS_FIFO_MM
int test_fifo(struct vfpga *f)
int fpga_test_hls_dft(struct vfpga *f)
{
struct ip *hls, *sw, *dma;
hls = ip_vlnv_lookup(&f->ips, NULL, "hls", NULL, NULL);
sw = ip_vlnv_lookup(&f->ips, "xilinx.com", "ip", "axis_interconnect", NULL);
dma = ip_vlnv_lookup(&f->ips, "xilinx.com", "ip", "axi_dma", NULL);
/* Check if required IP is available on FPGA */
if (!dma || !hls || !dma)
return -1;
return 0;
}
int fpga_test_fifo(struct vfpga *f)
{
int ret;
ssize_t len;
uintptr_t baseaddr = (uintptr_t) (f->map + BASEADDR_FIFO_MM);
char src[255], dst[255];
struct ip *fifo, *sw;
uint32_t ier;
fifo = ip_vlnv_lookup(&f->ips, "xilinx.com", "ip", "axi_fifo_mm", NULL);
if (!fifo)
return -1;
sw = ip_vlnv_lookup(&f->ips, "xilinx.com", "ip", "axis_interconnect", NULL);
if (!sw)
return -1;
/* Save old IER */
ier = XIntc_In32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET);
XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier | (1 << MSI_FIFO_MM));
ier = XIntc_In32(f->baseaddr.intc + XIN_IER_OFFSET);
XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier | (1 << fifo->irq));
/* Get some random data to compare */
memset(dst, 0, sizeof(dst));
@ -110,38 +176,48 @@ int test_fifo(struct vfpga *f)
if (ret)
error("Failed to get random data");
ret = switch_connect(&f->sw, AXIS_SWITCH_FIFO_MM, AXIS_SWITCH_FIFO_MM);
ret = switch_connect(&sw->sw, fifo->port, fifo->port);
if (ret)
error("Failed to configure switch");
len = fifo_write(&f->fifo, (char *) src, sizeof(src), f->vd.msi_efds[MSI_FIFO_MM]);
len = fifo_write(&fifo->fifo_mm, (char *) src, sizeof(src), f->vd.msi_efds[fifo->irq]);
if (len != sizeof(src))
error("Failed to send to FIFO");
len = fifo_read(&f->fifo, (char *) dst, sizeof(dst), f->vd.msi_efds[MSI_FIFO_MM]);
len = fifo_read(&fifo->fifo_mm, (char *) dst, sizeof(dst), f->vd.msi_efds[fifo->irq]);
if (len != sizeof(dst))
error("Failed to read from FIFO");
/* Restore IER */
XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier);
XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier);
/* Compare data */
return memcmp(src, dst, sizeof(src));
}
#endif
#if defined(HAS_DMA_SG) || defined(HAS_DMA_SIMPLE)
int test_dma(struct vfpga *f)
int fpga_test_dma(struct vfpga *f)
{
int ret;
struct ip *dma, *sw;
ssize_t len;
int ret;
uint32_t ier;
/* Save old IER */
ier = XIntc_In32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET);
XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier | (1 << MSI_DMA_MM2S) | (1 << MSI_DMA_S2MM));
dma = ip_vlnv_lookup(&f->ips, "xilinx.com", "ip", "axi_dma", NULL);
if (!dma)
return -1;
ret = switch_connect(&f->sw, AXIS_SWITCH_DMA_SIMPLE, AXIS_SWITCH_DMA_SIMPLE);
sw = ip_vlnv_lookup(&f->ips, "xilinx.com", "ip", "axis_interconnect", NULL);
if (!sw)
return -1;
int irq_mm2s = dma->irq;
int irq_s2mm = dma->irq + 1;
/* Save old IER */
ier = XIntc_In32(f->baseaddr.intc + XIN_IER_OFFSET);
XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier | (1 << irq_mm2s) | (1 << irq_s2mm));
ret = switch_connect(&sw->sw, dma->port, dma->port);
if (ret)
error("Failed to configure switch");
@ -161,60 +237,62 @@ int test_dma(struct vfpga *f)
#endif
/* Get some random data to compare */
char *src = f->dma;
char *dst = f->dma + TEST_LEN;
char *src = (char *) f->dma;
char *dst = (char *) f->dma + TEST_LEN;
ret = read_random(src, TEST_LEN);
if (ret)
error("Failed to get random data");
#if 0
len = dma_write(&f->dma_simple, virt_to_dma(src, f->dma), TEST_LEN, -1);//f->vd.msi_efds[MSI_DMA_MM2S]);
len = dma_write(&f->dma_simple, virt_to_dma(src, f->dma), TEST_LEN, -1);//f->vd.msi_efds[irq_mm2s]);
if (len != TEST_LEN)
error("Failed to send to DMAC: %zd", len);
len = dma_read(&f->dma_simple, virt_to_dma(dst, f->dma), TEST_LEN, -1);//f->vd.msi_efds[MSI_DMA_S2MM]);
len = dma_read(&f->dma_simple, virt_to_dma(dst, f->dma), TEST_LEN, -1);//f->vd.msi_efds[irq_s2mm]);
if (len != TEST_LEN)
error("Failed to send to DMAC: %zd", len);
#else
len = dma_ping_pong(&f->dma_simple, virt_to_dma(src, f->dma), virt_to_dma(dst, f->dma), TEST_LEN, f->vd.msi_efds[MSI_DMA_S2MM]);
len = dma_ping_pong(&dma->dma, virt_to_dma(src, f->dma), virt_to_dma(dst, f->dma), TEST_LEN, f->vd.msi_efds[irq_s2mm]);
if (len != TEST_LEN)
error("Failed to to ping pong DMA transfer: %zd", len);
#endif
/* Restore IER */
XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier);
XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier);
/* Check received data */
return memcmp(src, dst, TEST_LEN);
}
#endif
#ifdef HAS_TIMER
int test_timer(struct vfpga *f)
int fpga_test_timer(struct vfpga *f)
{
int ret;
bool success;
struct ip *timer;
uint32_t ier;
/* Save old IER */
ier = XIntc_In32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET);
XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier | (1 << MSI_TMRCTR0));
timer = ip_vlnv_lookup(&f->ips, "xilinx.com", "ip", "axi_timer", NULL);
if (!timer)
return -1;
XTmrCtr_SetOptions(&f->tmr, 0, XTC_EXT_COMPARE_OPTION | XTC_DOWN_COUNT_OPTION);
XTmrCtr_SetResetValue(&f->tmr, 0, AXI_HZ / 125);
XTmrCtr_Start(&f->tmr, 0);
/* Save old IER */
ier = XIntc_In32(f->baseaddr.intc + XIN_IER_OFFSET);
XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier | (1 << timer->irq));
XTmrCtr_SetOptions(&timer->timer, 0, XTC_EXT_COMPARE_OPTION | XTC_DOWN_COUNT_OPTION);
XTmrCtr_SetResetValue(&timer->timer, 0, AXI_HZ / 125);
XTmrCtr_Start(&timer->timer, 0);
struct pollfd pfd = {
.fd = f->vd.msi_efds[MSI_TMRCTR0],
.fd = f->vd.msi_efds[timer->irq],
.events = POLLIN
};
ret = poll(&pfd, 1, 1000);
if (ret == 1) {
uint64_t counter = wait_irq(f->vd.msi_efds[MSI_TMRCTR0]);
uint64_t counter = wait_irq(f->vd.msi_efds[timer->irq]);
info("Got IRQ: counter = %llu", counter);
info("Got IRQ: counter = %ju", counter);
if (counter == 1)
return 0;
@ -223,41 +301,51 @@ int test_timer(struct vfpga *f)
}
/* Restore IER */
XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier);
XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier);
return -1;
}
#endif
#ifdef HAS_RTDS_AXIS
int test_rtds_cbuilder(struct vfpga *f)
int fpga_test_rtds_cbuilder(struct vfpga *f)
{
#if 0
int ret;
int values_tx = 1;
int values_rx;
uint32_t tsc = 0, ovfl = 0;
ssize_t len;
uint32_t ier;
struct ip *rtds, *dma;
int irq_rtds_ovf = rtds->irq + 1;
rtds = ip_vlnv_lookup(&f->ips, "acs.eonerc.rwth-aachen.de", "user", "rtds_axis", NULL);
if (!rtds)
return -1;
dma = ip_vlnv_lookup(&f->ips, "xilinx.com", "ip", "axi_dma", NULL);
if (!dma)
return -1;
/* Save old IER */
ier = XIntc_In32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET);
// XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier | (1 << MSI_DMA_MM2S) | (1 << MSI_DMA_S2MM));
XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier | (1 << MSI_RTDS_OVF));
ier = XIntc_In32(f->baseaddr.intc + XIN_IER_OFFSET);
// XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier | (1 << MSI_DMA_MM2S) | (1 << MSI_DMA_S2MM));
XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier | (1 << irq_rtds_ovf);
/* Dump RTDS AXI Stream state */
rtds_axis_dump(f->map + BASEADDR_RTDS);
rtds_axis_dump(f->map + rtds->baseaddr);
/* Get some random data to compare */
float *data_rx = (uint32_t *) (f->dma);
float *data_tx = (uint32_t *) (f->dma + 0x1000);
/* Setup crossbar switch */
switch_connect(&f->sw, AXIS_SWITCH_RTDS, AXIS_SWITCH_DMA_SIMPLE);
switch_connect(&f->sw, AXIS_SWITCH_DMA_SIMPLE, AXIS_SWITCH_RTDS);
switch_connect(&sw->sw, rtds->port, dma->port);
switch_connect(&sw->sw, dma->port, rtds->port);
/* Disable blocking Overflow status */
int flags = fcntl(f->vd.msi_efds[MSI_RTDS_OVF], F_GETFL, 0);
fcntl(f->vd.msi_efds[MSI_RTDS_OVF], F_SETFL, flags | O_NONBLOCK);
int flags = fcntl(f->vd.msi_efds[irq_rtds_ovf], F_GETFL, 0);
fcntl(f->vd.msi_efds[irq_rtds_ovf], F_SETFL, flags | O_NONBLOCK);
/* Initialize CBuilder model */
double mdl_dt = rtds_axis_dt(f->map + BASEADDR_RTDS);
@ -279,9 +367,8 @@ int test_rtds_cbuilder(struct vfpga *f)
break;
/* Read data from RTDS */
len = dma_read(&f->dma_simple, virt_to_dma(data_rx, f->dma), 0x1000, -1);
uint64_t start = rdtscp();
XAxiDma_IntrAckIrq(&f->dma_simple, XAXIDMA_IRQ_IOC_MASK, XAXIDMA_DEVICE_TO_DMA);
len = dma_read(&dma->dma, virt_to_dma(data_rx, f->dma), 0x1000, -1);
XAxiDma_IntrAckIrq(&dma->dma, XAXIDMA_IRQ_IOC_MASK, XAXIDMA_DEVICE_TO_DMA);
values_rx = len / sizeof(uint32_t);
@ -289,32 +376,30 @@ int test_rtds_cbuilder(struct vfpga *f)
/* Run CBuilder model */
cbmodel_step(mdl_inputs, mdl_outputs);
data_tx[0] = mdl_outputs[0]; /* cast to float */
/* Send data to RTDS */
len = dma_write(&f->dma_simple, virt_to_dma(data_tx, f->dma), sizeof(uint32_t) * values_tx, -1);
XAxiDma_IntrAckIrq(&f->dma_simple, XAXIDMA_IRQ_IOC_MASK, XAXIDMA_DMA_TO_DEVICE);
len = dma_write(&dma->dma, virt_to_dma(data_tx, f->dma), sizeof(uint32_t) * values_tx, -1);
XAxiDma_IntrAckIrq(&dma->dma, XAXIDMA_IRQ_IOC_MASK, XAXIDMA_DMA_TO_DEVICE);
}
/* Restore IER */
XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier);
XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier);
#endif
return 0;
}
int test_rtds_rtt(struct vfpga *f)
int fpga_test_rtds_rtt(struct vfpga *f)
{
int ret;
int values_tx = 64;
int values_rx;
uint32_t tsc = 0, ovfl = 0;
#if 0
uint32_t tsc = 0;
ssize_t len;
uint32_t ier;
/* Save old IER */
ier = XIntc_In32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET);
// XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier | (1 << MSI_DMA_MM2S) | (1 << MSI_DMA_S2MM));
ier = XIntc_In32(f->baseaddr.intc + XIN_IER_OFFSET);
// XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier | (1 << MSI_DMA_MM2S) | (1 << MSI_DMA_S2MM));
/* Dump RTDS AXI Stream state */
rtds_axis_dump(f->map + BASEADDR_RTDS);
@ -397,8 +482,7 @@ retry: len = fifo_read(&f->fifo, (char *) data_rx, 0x1000, irq_fifo);
}
/* Restore IER */
XIntc_Out32((uintptr_t) f->map + BASEADDR_INTC + XIN_IER_OFFSET, ier);
XIntc_Out32(f->baseaddr.intc + XIN_IER_OFFSET, ier);
#endif
return 0;
}
#endif

148
src/fpga.c
View file

@ -9,10 +9,8 @@
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <dirent.h>
#include <string.h>
#include <errno.h>
#include <sys/mman.h>
#include <fcntl.h>
#include <getopt.h>
#include <sched.h>
@ -26,59 +24,59 @@
#include "config.h"
#include "config-fpga.h"
#include "fpga-tests.h"
#include "fpga-bench.h"
static struct pci_access *pacc;
/* Declarations */
int fpga_bench(struct vfpga *f);
int fpga_tests(struct vfpga *f);
void usage(char *name)
{
printf("Usage: %s [ARGS]\n", name);
printf(" -d Set log level\n");
printf(" -i ID pair of PCI device: [vendor]:[device][:class]\n");
printf(" -s Slot of PCI device: [[[domain]:][bus]:][slot][.[func]]\n\n");
printf("fpga PCIutil %s (built on %s %s)\n",
VERSION, __DATE__, __TIME__);
printf(" copyright 2014-2015, Institute for Automation of Complex Power Systems, EONERC\n");
printf(" Steffen Vogel <StVogel@eonerc.rwth-aachen.de>\n");
printf("Usage: %s CONFIGFILE CMD [OPTIONS]\n", name);
printf(" Commands:\n");
printf(" tests Test functionality of VILLASfpga card\n");
printf(" bench Do benchmarks\n\n");
printf(" Options:\n");
printf(" -d Set log level\n\n");
print_copyright();
exit(EXIT_FAILURE);
}
int main(int argc, char *argv[])
{
int ret;
char *err;
struct vfpga fpga;
struct vfio_container vc;
struct pci_dev *pdev;
struct pci_filter filter;
/* Init libpci */
pacc = pci_alloc(); /* Get the pci_access structure */
pci_init(pacc); /* Initialize the PCI library */
pci_scan_bus(pacc); /* We want to get the list of devices */
pci_filter_init(pacc, &filter);
/* Default filter */
filter.vendor = PCI_VID_XILINX;
filter.device = PCI_PID_VFPGA;
struct vfpga *fpga;
enum {
FPGA_TESTS,
FPGA_BENCH
} subcommand;
config_t config;
if (argc < 3)
usage(argv[0]);
if (strcmp(argv[2], "tests") == 0)
subcommand = FPGA_TESTS;
else if (strcmp(argv[2], "bench") == 0)
subcommand = FPGA_BENCH;
else
usage(argv[0]);
/* Setup libconfig */
config_init(&config);
ret = config_read_file(&config, argv[1]);
if (ret != CONFIG_TRUE) {
error("Failed to parse configuration: %s in %s:%d",
config_error_text(&config),
config_error_file(&config) ? config_error_file(&config) : argv[1],
config_error_line(&config)
);
}
/* Parse arguments */
char c, *endptr;
while ((c = getopt (argc, argv, "i:s:d:")) != -1) {
while ((c = getopt (argc-1, argv+1, "d:")) != -1) {
switch (c) {
case 's':
err = pci_filter_parse_slot(&filter, optarg);
if (err)
error("Failed to parse slot: %s", err);
break;
case 'i':
err = pci_filter_parse_id(&filter, optarg);
if (err)
error("Failed to parse ID: %s", err);
break;
case 'd':
log_setlevel(strtoul(optarg, &endptr, 10), ~0);
break;
@ -89,22 +87,15 @@ int main(int argc, char *argv[])
}
}
/* Search for fpga card */
pdev = pci_find_device(pacc, &filter);
if (!pdev)
error("Failed to find PCI device");
/* Get VFIO handles and details */
ret = vfio_init(&vc);
if (ret)
serror("Failed to initialize VFIO");
/* Initialize fpga card */
ret = vfpga_init2(&fpga, &vc, pdev);
/* Initialize VILLASfpga card */
config_setting_t *cfg_root = config_root_setting(&config);
ret = vfpga_init(argc, argv, cfg_root);
if (ret)
error("Failed to initialize fpga card");
fpga = vfpga_get();
vfpga_dump(&fpga);
vfpga_dump(fpga);
/* Setup scheduler */
cpu_set_t set = integer_to_cpuset(AFFINITY);
@ -117,49 +108,22 @@ int main(int argc, char *argv[])
if (ret)
serror("Failed to change scheduler");
for (int i = 0; i < fpga.vd.irqs[VFIO_PCI_MSI_IRQ_INDEX].count; i++) {
ret = kernel_irq_setaffinity(fpga.vd.msi_irqs[i], AFFINITY, NULL);
for (int i = 0; i < fpga->vd.irqs[VFIO_PCI_MSI_IRQ_INDEX].count; i++) {
ret = kernel_irq_setaffinity(fpga->vd.msi_irqs[i], AFFINITY, NULL);
if (ret)
serror("Failed to change affinity of VFIO-MSI interrupt");
}
/* Start tests */
ret = test_intc(&fpga);
info("INTC Test: %s", (ret == 0) ? GRN("passed") : RED("failed"));
ret = test_xsg(&fpga, "xsg_multiply_add");
info("XSG Test: %s", (ret == 0) ? GRN("passed") : RED("failed"));
// ret = test_timer(&fpga);
// info("Timer Test: %s", (ret == 0) ? GRN("passed") : RED("failed"));
// ret = test_fifo(&fpga);
// info("FIFO Test: %s", (ret == 0) ? GRN("passed") : RED("failed"));
ret = test_dma(&fpga);
info("DMA Test: %s", (ret == 0) ? GRN("passed") : RED("failed"));
// ret = test_rtds_rtt(&fpga);
// info("RTDS RTT Test: %s", (ret == 0) ? GRN("passed") : RED("failed"));
// ret = test_rtds_cbuilder(&fpga);
// info("RTDS RTT Test: %s", (ret == 0) ? GRN("passed") : RED("failed"));
/* Start benchmarks */
#if 0
ret = bench_memcpy(&fpga);
#endif
/* Start subcommand */
switch (subcommand) {
case FPGA_TESTS: fpga_tests(fpga); break;
case FPGA_BENCH: /*fpga_bench(fpga);*/ break;
}
/* Shutdown */
ret = vfpga_deinit2(&fpga);
ret = vfpga_deinit(&fpga);
if (ret)
error("Failed to de-initialize fpga card");
ret = vfio_destroy(&vc);
if (ret)
error("Failed to deinitialize VFIO module");
pci_cleanup(pacc);
return 0;
}