From abeaa0b0779991caa23a09438e62c9af0c24e4f3 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Tue, 30 Jan 2018 18:27:13 +0100 Subject: [PATCH 1/9] execute FPGA unit on acs-villas --- fpga/.gitlab-ci.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/fpga/.gitlab-ci.yml b/fpga/.gitlab-ci.yml index 6bacbe506..5575b86f6 100644 --- a/fpga/.gitlab-ci.yml +++ b/fpga/.gitlab-ci.yml @@ -69,6 +69,20 @@ test:unit-software: tags: - docker +test:unit-hardware: + stage: test + dependencies: + - build:source + tags: + - villas-fpga + script: | + if [ "$(who | wc -l)" -eq "0" ]; then + build/tests/unit-tests --filter 'fpga/*' + else + echo "System is currently used by: $(who)" + echo "We are skipping the test. Please restart manually." + fi + # Stage: deploy ############################################################################## From 269550c5dcf681fbc139acaa6d56d6b00b323149 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Tue, 30 Jan 2018 18:34:27 +0100 Subject: [PATCH 2/9] install libraries to fix loading of libvillas-fpga.so --- fpga/.gitlab-ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/fpga/.gitlab-ci.yml b/fpga/.gitlab-ci.yml index 5575b86f6..2136dea25 100644 --- a/fpga/.gitlab-ci.yml +++ b/fpga/.gitlab-ci.yml @@ -71,13 +71,15 @@ test:unit-software: test:unit-hardware: stage: test - dependencies: - - build:source +# dependencies: +# - build:source tags: - villas-fpga script: | + mkdir build && cd build && cmake .. && make unit-tests + if [ "$(who | wc -l)" -eq "0" ]; then - build/tests/unit-tests --filter 'fpga/*' + tests/unit-tests --filter 'fpga/*' else echo "System is currently used by: $(who)" echo "We are skipping the test. Please restart manually." From 293f496db088f4a674f7246f64ca2968cfeab435 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Tue, 30 Jan 2018 19:09:19 +0100 Subject: [PATCH 3/9] pci: add function to get currently loaded kernel driver --- fpga/include/villas/kernel/pci.h | 3 +++ fpga/lib/kernel/pci.c | 21 ++++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/fpga/include/villas/kernel/pci.h b/fpga/include/villas/kernel/pci.h index d784cedf8..db6beb0e6 100644 --- a/fpga/include/villas/kernel/pci.h +++ b/fpga/include/villas/kernel/pci.h @@ -57,6 +57,9 @@ int pci_device_compare(const struct pci_device *d, const struct pci_device *f); struct pci_device * pci_lookup_device(struct pci *p, struct pci_device *filter); +/** Get currently loaded driver for device */ +int pci_get_driver(struct pci_device *d, char *buf, size_t buflen); + /** Bind a new LKM to the PCI device */ int pci_attach_driver(struct pci_device *d, const char *driver); diff --git a/fpga/lib/kernel/pci.c b/fpga/lib/kernel/pci.c index 56a8c17bd..e4d6793ba 100644 --- a/fpga/lib/kernel/pci.c +++ b/fpga/lib/kernel/pci.c @@ -254,10 +254,29 @@ struct pci_device * pci_lookup_device(struct pci *p, struct pci_device *f) return list_search(&p->devices, (cmp_cb_t) pci_device_compare, (void *) f); } +int pci_get_driver(struct pci_device *d, char *buf, size_t buflen) +{ + int ret; + char sysfs[1024], syml[1024]; + + snprintf(sysfs, sizeof(sysfs), "%s/bus/pci/devices/%04x:%02x:%02x.%x/driver", SYSFS_PATH, + d->slot.domain, d->slot.bus, d->slot.device, d->slot.function); + + ret = readlink(sysfs, syml, sizeof(syml)); + if (ret < 0) + return ret; + + char *driver = basename(syml); + + strncpy(buf, driver, buflen); + + return 0; +} + int pci_attach_driver(struct pci_device *d, const char *driver) { FILE *f; - char fn[256]; + char fn[1024]; /* Add new ID to driver */ snprintf(fn, sizeof(fn), "%s/bus/pci/drivers/%s/new_id", SYSFS_PATH, driver); From 3047f5bb7a2b34646c6b587c83da13b84401803b Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Tue, 30 Jan 2018 19:09:56 +0100 Subject: [PATCH 4/9] vfio: only rebind pci device to VFIO driver if not already bound --- fpga/lib/kernel/vfio.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/fpga/lib/kernel/vfio.c b/fpga/lib/kernel/vfio.c index 018f42dcf..0c0de58d4 100644 --- a/fpga/lib/kernel/vfio.c +++ b/fpga/lib/kernel/vfio.c @@ -229,10 +229,13 @@ int vfio_pci_attach(struct vfio_device *d, struct vfio_container *c, struct pci_ if (kernel_module_load("vfio_pci")) error("Failed to load kernel driver: %s", "vfio_pci"); - /* Bind PCI card to vfio-pci driver*/ - ret = pci_attach_driver(pdev, "vfio-pci"); - if (ret) - error("Failed to attach device to driver"); + /* Bind PCI card to vfio-pci driver if not already bound */ + ret = pci_get_driver(pdev, name, sizeof(name)); + if (ret || strcmp(name, "vfio-pci")) { + ret = pci_attach_driver(pdev, "vfio-pci"); + if (ret) + error("Failed to attach device to driver"); + } /* Get IOMMU group of device */ int index = pci_get_iommu_group(pdev); From 4f86b98fdd7c6dda813b9f9380780a0b010c55df Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Tue, 30 Jan 2018 19:15:45 +0100 Subject: [PATCH 5/9] add script to configure system for non-root access to FPGA --- fpga/scripts/non_root.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 fpga/scripts/non_root.sh diff --git a/fpga/scripts/non_root.sh b/fpga/scripts/non_root.sh new file mode 100644 index 000000000..22e4f941d --- /dev/null +++ b/fpga/scripts/non_root.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +IOMMU_GROUP=24 +PCI_BDF="0000:03:00.0" + +modprobe vfio +modprobe vfio_pci + +echo "10ee 7022" > /sys/bus/pci/drivers/vfio-pci/new_id +echo ${PCI_BDF} > /sys/bus/pci/drivers/vfio-pci/bind + +groupadd -f fpga +usermod -G fpga -a svg + +chgrp fpga /dev/vfio/${IOMMU_GROUP} +chmod g+rw /dev/vfio/${IOMMU_GROUP} + + From a45de4ec1a496fe2c9b5276e5dbde77c7875c807 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Tue, 30 Jan 2018 19:27:22 +0100 Subject: [PATCH 6/9] FPGA tests fail if we attempt to run them in parallel --- fpga/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fpga/.gitlab-ci.yml b/fpga/.gitlab-ci.yml index 2136dea25..236d26978 100644 --- a/fpga/.gitlab-ci.yml +++ b/fpga/.gitlab-ci.yml @@ -79,7 +79,7 @@ test:unit-hardware: mkdir build && cd build && cmake .. && make unit-tests if [ "$(who | wc -l)" -eq "0" ]; then - tests/unit-tests --filter 'fpga/*' + tests/unit-tests --jobs 1 --filter 'fpga/*' else echo "System is currently used by: $(who)" echo "We are skipping the test. Please restart manually." From 4a4a5fe17643dd2b61879e90204340cc6c2d5a1f Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Tue, 30 Jan 2018 19:32:22 +0100 Subject: [PATCH 7/9] tiny change --- fpga/.gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/fpga/.gitlab-ci.yml b/fpga/.gitlab-ci.yml index 236d26978..f623edf97 100644 --- a/fpga/.gitlab-ci.yml +++ b/fpga/.gitlab-ci.yml @@ -77,7 +77,6 @@ test:unit-hardware: - villas-fpga script: | mkdir build && cd build && cmake .. && make unit-tests - if [ "$(who | wc -l)" -eq "0" ]; then tests/unit-tests --jobs 1 --filter 'fpga/*' else From e25a3e24bc52d4706547460a874e4fd92e1669f8 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Tue, 30 Jan 2018 19:35:03 +0100 Subject: [PATCH 8/9] cleanup build dir before building --- fpga/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fpga/.gitlab-ci.yml b/fpga/.gitlab-ci.yml index f623edf97..b45b20877 100644 --- a/fpga/.gitlab-ci.yml +++ b/fpga/.gitlab-ci.yml @@ -76,7 +76,7 @@ test:unit-hardware: tags: - villas-fpga script: | - mkdir build && cd build && cmake .. && make unit-tests + rm -r build && mkdir build && cd build && cmake .. && make unit-tests if [ "$(who | wc -l)" -eq "0" ]; then tests/unit-tests --jobs 1 --filter 'fpga/*' else From 3c7f6e968b51f01eb64144f8d16095090e695236 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Tue, 30 Jan 2018 19:36:06 +0100 Subject: [PATCH 9/9] do parallel build --- fpga/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fpga/.gitlab-ci.yml b/fpga/.gitlab-ci.yml index b45b20877..ab5b0d02e 100644 --- a/fpga/.gitlab-ci.yml +++ b/fpga/.gitlab-ci.yml @@ -33,7 +33,7 @@ docker-dev: build:source: stage: build script: - - mkdir build && cd build && cmake .. && make + - mkdir build && cd build && cmake .. && make -j8 artifacts: expire_in: 1 week name: ${CI_PROJECT_NAME}-${CI_BUILD_REF} @@ -76,7 +76,7 @@ test:unit-hardware: tags: - villas-fpga script: | - rm -r build && mkdir build && cd build && cmake .. && make unit-tests + rm -r build && mkdir build && cd build && cmake .. && make unit-tests -j8 if [ "$(who | wc -l)" -eq "0" ]; then tests/unit-tests --jobs 1 --filter 'fpga/*' else