Refactor FPGA device and drivers

Signed-off-by: Pascal Bauer <pascal.bauer@rwth-aachen.de>

common/include/villas/kernel/devices/device.hpp

@@ -0,0 +1,33 @@
+/* Interface for Linux/Unix devices.
+ *
+ * Author: Pascal Bauer <pascal.bauer@rwth-aachen.de>
+ *
+ * SPDX-FileCopyrightText: 2023-2024 Pascal Bauer <pascal.bauer@rwth-aachen.de>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+#include <filesystem>
+#include <optional>
+#include <villas/kernel/devices/driver.hpp>
+
+namespace villas {
+namespace kernel {
+namespace devices {
+
+class Device {
+public:
+  virtual ~Device(){};
+
+  virtual std::optional<std::unique_ptr<Driver>> driver() const = 0;
+  virtual std::optional<int> iommu_group() const = 0;
+  virtual std::string name() const = 0;
+  virtual std::filesystem::path override_path() const = 0;
+  virtual std::filesystem::path path() const = 0;
+  virtual void probe() const = 0;
+};
+
+} // namespace devices
+} // namespace kernel
+} // namespace villas

common/include/villas/kernel/devices/driver.hpp

@@ -0,0 +1,29 @@
+/* Interface for device drivers. OS/platform independend. 
+ * Implemented for Linux/Unix drivers in linux_driver.hpp
+ *
+ * Author: Pascal Bauer <pascal.bauer@rwth-aachen.de>
+ *
+ * SPDX-FileCopyrightText: 2023-2024 Pascal Bauer <pascal.bauer@rwth-aachen.de>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+namespace villas {
+namespace kernel {
+namespace devices {
+
+class Device;
+
+class Driver {
+public:
+  virtual void attach(const Device &device) const = 0;
+  virtual void bind(const Device &device) const = 0;
+  virtual std::string name() const = 0;
+  virtual void override(const Device &device) const = 0;
+  virtual void unbind(const Device &device) const = 0;
+};
+
+} // namespace devices
+} // namespace kernel
+} // namespace villas

common/include/villas/kernel/devices/ip_device.hpp

@@ -0,0 +1,35 @@
+/* Linux/Unix device which represents an IP component of a FPGA.
+ *
+ * Author: Pascal Bauer <pascal.bauer@rwth-aachen.de>
+ *
+ * SPDX-FileCopyrightText: 2023-2024 Pascal Bauer <pascal.bauer@rwth-aachen.de>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+#include <filesystem>
+#include <villas/kernel/devices/platform_device.hpp>
+
+namespace villas {
+namespace kernel {
+namespace devices {
+
+class IpDevice : public PlatformDevice {
+public:
+  static IpDevice from(const std::filesystem::path unsafe_path);
+  static bool is_path_valid(const std::filesystem::path unsafe_path);
+
+private:
+  IpDevice() = delete;
+  IpDevice(const std::filesystem::path valid_path) //! Dont allow unvalidated paths
+      : PlatformDevice(valid_path){};
+
+public:
+  size_t addr() const;
+  std::string ip_name() const;
+};
+
+} // namespace devices
+} // namespace kernel
+} // namespace villas

common/include/villas/kernel/devices/linux_driver.hpp

@@ -0,0 +1,52 @@
+/* Implementation of driver interface for Linux/Unix based operation system drivers.
+ *
+ * Author: Pascal Bauer <pascal.bauer@rwth-aachen.de>
+ *
+ * SPDX-FileCopyrightText: 2023-2024 Pascal Bauer <pascal.bauer@rwth-aachen.de>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <villas/kernel/devices/driver.hpp>
+
+namespace villas {
+namespace kernel {
+namespace devices {
+
+class LinuxDriver : public Driver {
+private:
+  static constexpr char BIND_DEFAULT[] = "bind";
+  static constexpr char UNBIND_DEFAULT[] = "unbind";
+
+public:
+  const std::filesystem::path path;
+
+private:
+  const std::filesystem::path bind_path;
+  const std::filesystem::path unbind_path;
+
+public:
+  LinuxDriver(const std::filesystem::path path)
+      : LinuxDriver(path, path / std::filesystem::path(BIND_DEFAULT),
+                    path / std::filesystem::path(UNBIND_DEFAULT)){};
+
+  LinuxDriver(const std::filesystem::path path,
+              const std::filesystem::path bind_path,
+              const std::filesystem::path unbind_path)
+      : path(path), bind_path(bind_path), unbind_path(unbind_path){};
+
+public:
+  void attach(const Device &device) const override;
+  void bind(const Device &device) const override;
+  std::string name() const override;
+  void override(const Device &device) const override;
+  void unbind(const Device &device) const override;
+};
+
+} // namespace devices
+} // namespace kernel
+} // namespace villas

common/include/villas/kernel/devices/platform_device.hpp

@@ -0,0 +1,51 @@
+/* Platform bus based Linux/Unix device.
+ *
+ * Author: Pascal Bauer <pascal.bauer@rwth-aachen.de>
+ *
+ * SPDX-FileCopyrightText: 2023-2024 Pascal Bauer <pascal.bauer@rwth-aachen.de>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+#include <filesystem>
+#include <villas/kernel/devices/device.hpp>
+#include <villas/kernel/devices/driver.hpp>
+
+namespace villas {
+namespace kernel {
+namespace devices {
+
+class PlatformDevice : public Device {
+private:
+  static constexpr char PROBE_DEFAULT[] = "/sys/bus/platform/drivers_probe";
+  static constexpr char OVERRIDE_DEFAULT[] = "driver_override";
+
+private:
+  const std::filesystem::path m_path;
+  const std::filesystem::path m_probe_path;
+  const std::filesystem::path m_override_path;
+
+public:
+  PlatformDevice(const std::filesystem::path path)
+      : PlatformDevice(path, std::filesystem::path(PROBE_DEFAULT),
+                       path / std::filesystem::path(OVERRIDE_DEFAULT)){};
+
+  PlatformDevice(const std::filesystem::path path,
+                 const std::filesystem::path probe_path,
+                 const std::filesystem::path override_path)
+      : m_path(path), m_probe_path(probe_path),
+        m_override_path(override_path){};
+
+  // Implement device interface
+  std::optional<std::unique_ptr<Driver>> driver() const override;
+  std::optional<int> iommu_group() const override;
+  std::string name() const override;
+  std::filesystem::path override_path() const override;
+  std::filesystem::path path() const override;
+  void probe() const override;
+};
+
+} // namespace devices
+} // namespace kernel
+} // namespace villas

common/include/villas/utils.hpp

@@ -11,6 +11,7 @@
 #include <list>
 #include <string>
 #include <vector>
+#include <filesystem>
 
 #include <cassert>
 #include <cstdint>
@@ -208,9 +209,12 @@ template <class... Ts> struct overloaded : Ts... {
   using Ts::operator()...;
 };
 
-// explicit deduction guide (not needed as of C++20)
+// Explicit deduction guide (not needed as of C++20)
 template <class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
 
+void write_to_file(std::string data, const std::filesystem::path file);
+std::vector<std::string> read_names_in_directory(const std::filesystem::path &directory);
+
 namespace base64 {
 
 using byte = std::uint8_t;

common/lib/CMakeLists.txt

@@ -39,7 +39,10 @@ endif()
 
 if(CMAKE_SYSTEM_NAME STREQUAL Linux)
     target_sources(villas-common PRIVATE
+        kernel/devices/ip_device.cpp
+        kernel/devices/linux_driver.cpp
         kernel/devices/pci_device.cpp
+        kernel/devices/platform_device.cpp
         kernel/vfio_device.cpp
         kernel/vfio_group.cpp
         kernel/vfio_container.cpp

common/lib/kernel/devices/ip_device.cpp

@@ -0,0 +1,54 @@
+/* Linux/Unix device which represents an IP component of a FPGA.
+ *
+ * Author: Pascal Bauer <pascal.bauer@rwth-aachen.de>
+ *
+ * SPDX-FileCopyrightText: 2023-2024 Pascal Bauer <pascal.bauer@rwth-aachen.de>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <filesystem>
+#include <regex>
+#include <stdexcept>
+
+#include <villas/exceptions.hpp>
+#include <villas/kernel/devices/ip_device.hpp>
+
+using villas::kernel::devices::IpDevice;
+
+IpDevice IpDevice::from(const std::filesystem::path unsafe_path) {
+  if (!is_path_valid(unsafe_path))
+    throw RuntimeError(
+        "Path {} failed validation as IpDevicePath [adress in hex].[name] ",
+        unsafe_path.u8string());
+  return IpDevice(unsafe_path);
+}
+
+std::string IpDevice::ip_name() const {
+  int pos = name().find('.');
+  return name().substr(pos + 1);
+}
+
+size_t IpDevice::addr() const {
+  size_t pos = name().find('.');
+  std::string addr_hex = name().substr(0, pos);
+
+  // Convert from hex-string to number
+  std::stringstream ss;
+  ss << std::hex << addr_hex;
+  size_t addr = 0;
+  ss >> addr;
+
+  return addr;
+}
+
+bool IpDevice::is_path_valid(const std::filesystem::path unsafe_path) {
+  std::string assumed_device_name = unsafe_path.filename();
+
+  // Match format of hexaddr.devicename
+  if (!std::regex_match(assumed_device_name,
+                        std::regex(R"([0-9A-Fa-f]+\..*)"))) {
+    return false;
+  }
+
+  return true;
+}

common/lib/kernel/devices/linux_driver.cpp

@@ -0,0 +1,40 @@
+/* Implementation of driver interface for Linux/Unix based operation system drivers.
+ *
+ * Author: Pascal Bauer <pascal.bauer@rwth-aachen.de>
+ *
+ * SPDX-FileCopyrightText: 2023-2024 Pascal Bauer <pascal.bauer@rwth-aachen.de>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <villas/kernel/devices/linux_driver.hpp>
+
+#include <villas/kernel/devices/device.hpp>
+#include <villas/utils.hpp>
+
+using villas::kernel::devices::Device, villas::kernel::devices::LinuxDriver;
+using villas::utils::write_to_file;
+
+void LinuxDriver::attach(const Device &device) const {
+  if (device.driver().has_value()) {
+    device.driver().value()->unbind(device);
+  }
+  this->override(device);
+  device.probe();
+}
+
+void LinuxDriver::bind(const Device &device) const {
+  write_to_file(device.name(), this->bind_path);
+}
+
+std::string LinuxDriver::name() const {
+  size_t pos = path.u8string().rfind('/');
+  return path.u8string().substr(pos + 1);
+}
+
+void LinuxDriver::override(const Device &device) const {
+  write_to_file(this->name(), device.override_path());
+}
+
+void LinuxDriver::unbind(const Device &device) const {
+  write_to_file(device.name(), this->unbind_path);
+}

common/lib/kernel/devices/platform_device.cpp

@@ -0,0 +1,50 @@
+/* Platform bus based Linux/Unix device.
+ *
+ * Author: Pascal Bauer <pascal.bauer@rwth-aachen.de>
+ *
+ * SPDX-FileCopyrightText: 2023-2024 Pascal Bauer <pascal.bauer@rwth-aachen.de>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <villas/kernel/devices/linux_driver.hpp>
+#include <villas/kernel/devices/platform_device.hpp>
+#include <villas/utils.hpp>
+
+using villas::kernel::devices::Driver, villas::kernel::devices::LinuxDriver;
+using villas::kernel::devices::PlatformDevice;
+using villas::utils::write_to_file;
+
+std::optional<std::unique_ptr<Driver>> PlatformDevice::driver() const {
+  std::filesystem::path driver_symlink =
+      this->m_path / std::filesystem::path("driver");
+
+  if (!std::filesystem::is_symlink(driver_symlink))
+    return std::nullopt;
+
+  std::filesystem::path driver_path =
+      std::filesystem::canonical(driver_symlink);
+  return std::make_optional(std::make_unique<LinuxDriver>(driver_path));
+}
+
+std::optional<int> PlatformDevice::iommu_group() const {
+  std::filesystem::path symlink =
+      std::filesystem::path(this->m_path.u8string() + "/iommu_group");
+
+  std::filesystem::path link = std::filesystem::read_symlink(symlink);
+  std::string delimiter = "iommu_groups/";
+  int pos = link.u8string().find(delimiter);
+  int iommu_group = std::stoi(link.u8string().substr(pos + delimiter.length()));
+  return std::make_optional<int>(iommu_group);
+}
+
+std::filesystem::path PlatformDevice::path() const { return this->m_path; };
+
+void PlatformDevice::probe() const {
+  write_to_file(this->name(), this->m_probe_path);
+}
+
+std::filesystem::path PlatformDevice::override_path() const {
+  return this->m_override_path;
+}
+
+std::string PlatformDevice::name() const { return this->m_path.filename(); }

common/lib/utils.cpp

@@ -17,7 +17,11 @@
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
+#include <dirent.h>
 #include <fcntl.h>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
 #include <pthread.h>
 #include <unistd.h>
 
@@ -351,5 +355,27 @@ bool isPrivileged() {
   return true;
 }
 
+void write_to_file(std::string data, const std::filesystem::path file) {
+  villas::Log::get("Filewriter")->debug("{} > {}", data, file.u8string());
+  std::ofstream outputFile(file.u8string());
+
+  if (outputFile.is_open()) {
+    outputFile << data;
+    outputFile.close();
+  } else {
+    throw std::filesystem::filesystem_error("Cannot open outputfile",
+                                            std::error_code());
+  }
+}
+
+std::vector<std::string>
+read_names_in_directory(const std::filesystem::path &directory) {
+  std::vector<std::string> names;
+  for (auto const &dir_entry : std::filesystem::directory_iterator{directory}) {
+      names.push_back(dir_entry.path().filename());
+  }
+  return names;
+}
+
 } // namespace utils
 } // namespace villas