diff --git a/.editorconfig b/.editorconfig index 4d1d5187e..a75847bc8 100644 --- a/.editorconfig +++ b/.editorconfig @@ -20,3 +20,9 @@ trim_trailing_whitespace=true charset = utf-8 indent_style = tab indent_size = 8 + +[*.nix] +charset = utf-8 +indent_style = space +indent_size = 2 +trim_trailing_whitespace=true diff --git a/.envrc b/.envrc index 621e974a8..7239e2fce 100644 --- a/.envrc +++ b/.envrc @@ -1,7 +1,34 @@ + +export_or_unset() +{ + local var=$1 + + if [ -z "${!var+x}" ]; then + return + fi + + if [ -n "$2" ]; then + export $var="$2" + else + unset $var + fi + +} + if direnv_version "2.30.0" \ && has nix \ && nix show-config experimental-features 2>/dev/null | grep -wqF flakes then - watch_file ./packaging/nix/*.nix - use flake ./packaging/nix + local oldtmp="$TMP" + local oldtemp="$TEMP" + local oldtmpdir="$TMPDIR" + local oldtempdir="$TEMPDIR" + + watch_file ./packaging/nix/*.nix + use flake ./packaging/nix + + export_or_unset TMP "$oldtmp" + export_or_unset TEMP "$oldtemp" + export_or_unset TMPDIR "$oldtmpdir" + export_or_unset TEMPDIR "$oldtempdir" fi diff --git a/packaging/nix/README.md b/packaging/nix/README.md index 968ac41ff..6a12f48d9 100644 --- a/packaging/nix/README.md +++ b/packaging/nix/README.md @@ -56,10 +56,23 @@ You can also install `villas` into your local profile and have it available in nix profile install 'github:VILLASframework/node?dir=packaging/nix#villas' ``` +If you don't want to add it directly into the global path you could add it into +the flake registry as well. + +```shell +nix registy add 'github:VILLASframework/node?dir=packaging/nix#villas' +``` + +This allows you to substitue all references to +`github:VILLASframework/node?dir=packaging/nix#villas` with a simple `villas`. +I'll be using the `villas` registry entry in the following sections. + ## Development You can easily setup a development shell environment for `villas` by using the `devShells` provided in the [`flake.nix`] using `nix develop`. +Try for example these commands in the repository root to create a new shell with +all required dependecies to build various configurations of `villas`. ```shell # create a shell with all required build dependecies but without most optional ones @@ -87,6 +100,7 @@ images without worrying about missing dependencies or Copying things inbetween nothing else can be done using only a few lines of Nix. Here we build a docker image containing the `#villas` flake output: + ```shell # `docker load` reads OCI images from stdin # `nix build --impure --expr` builds a derivation from an impure Nix expression @@ -94,8 +108,7 @@ Here we build a docker image containing the `#villas` flake output: # `--print-out-paths` prints the built image's path to stdout docker load < $(nix build --no-link --print-out-paths --impure --expr ' let - ref = "github:VILLASframework/node?dir=packaging/nix"; - villas = (builtins.getFlake ref).packages.x86_64-linux.villas; + villas = (builtins.getFlake "villas").packages.x86_64-linux.villas; pkgs = (builtins.getFlake "nixpkgs").legacyPackages.x86_64-linux; in pkgs.dockerTools.buildImage { @@ -110,6 +123,40 @@ docker load < $(nix build --no-link --print-out-paths --impure --expr ' See https://nixos.org/manual/nixpkgs/stable/#sec-pkgs-dockerTools +## Simple cross-compilation to aarch64 + +The flake currently supports `x86_64-linux` and `aarch64-linux` systems. But +especially on some weaker `aarch64` target machines, compiling `villas` from +source is rather cumbersome. This flake has a non-standard `crossPackages` +flake output. This is currently only useful for cross-compiling from `x86_64` +to `aarch64`. + +```shell +nix build villas#crossPackages.x86_64-linux.aarch64-multiplatform.villas +``` + +If your target has a nix installation you can directly build and copy the +cross-compiled package using `nix copy`. + +```shell +# build and copy in one +nix copy villas#crossPackages.x86_64-linux.aarch64-multiplatform.villas --to ssh-ng://target-system +``` + +```shell +# build the cross-compiled package +nix build villas#crossPackages.x86_64-linux.aarch64-multiplatform.villas + +# copy it over +nix copy $(readlink result) --to ssh-ng://target + +# add the cross-compiled package to the target's PATH +ssh target nix profile add $(readlink result) +``` + +Further parameters for port or user of the ssh connection can only be specified +in the ssh configuration files. + ## Customization The [`villas.nix`] file contains the Nix "derivation" used to @@ -134,11 +181,10 @@ See https://nixos.org/manual/nixpkgs/stable/#chap-overrides # `pkgs` is the set of `x86_64-linux` packages provided by the flake nix build --impure --expr ' let - ref = "github:VILLASframework/node?dir=packaging/nix"; - pkgs = (builtins.getFlake ref).packages.x86_64-linux; + pkgs = (builtins.getFlake "villas").packages.x86_64-linux; in pkgs.villas-minimal.override { - withConfig = true; + withExtraConfig = true; withNodeIec60870 = true; } ' @@ -180,7 +226,8 @@ Here is a basic flake to build upon: packages.x86_64-linux = rec { default = villas-custom; villas-custom = villas-pkgs.villas-minimal.override { - withConfig = true; + version = "custom"; + withExtraConfig = true; withNodeIec60870 = true; }; }; @@ -228,7 +275,8 @@ typical Nix usage. A more interesting use of Nix would be a custom Nix shell environment containing your `villas-custom` and other tools of your choice. Here is a more complete `flake.nix` containing `devShells` available to -`nix develop` and an OCI image: +`nix develop` and an OCI image. + ```nix # flake.nix { diff --git a/packaging/nix/flake.lock b/packaging/nix/flake.lock index 78f077e45..de909cac6 100644 --- a/packaging/nix/flake.lock +++ b/packaging/nix/flake.lock @@ -3,11 +3,11 @@ "common": { "flake": false, "locked": { - "lastModified": 1680527215, - "narHash": "sha256-X62JISWeukC20t30XQbd9q0lf891zim7KqB3aRd0ieA=", + "lastModified": 1686671462, + "narHash": "sha256-yvpCp9eZO05lWWz/bYifDlalDTd9WjH84QwFxmsjoHg=", "owner": "VILLASframework", "repo": "common", - "rev": "d0b6edfe2a93297e0caf218738495686497a3dd6", + "rev": "d9d4ac76a5403e14f7899dae480781e9cdcf0572", "type": "github" }, "original": { @@ -16,6 +16,24 @@ "type": "github" } }, + "fpga": { + "flake": false, + "locked": { + "lastModified": 1679417867, + "narHash": "sha256-HHIsOnuQOq0Ytd22VMSHpTbecHpcheF1xLkslqxXOsk=", + "ref": "refs/heads/master", + "rev": "29f81016b2317667a2b21fbcf74ea66986a84c03", + "revCount": 530, + "submodules": true, + "type": "git", + "url": "https://github.com/VILLASframework/fpga.git" + }, + "original": { + "submodules": true, + "type": "git", + "url": "https://github.com/VILLASframework/fpga.git" + } + }, "lib60870": { "flake": false, "locked": { @@ -33,6 +51,25 @@ "type": "github" } }, + "libdatachannel": { + "flake": false, + "locked": { + "lastModified": 1683797946, + "narHash": "sha256-kSK+5gFMG6tq89R1m08gNBKPdwyR/mLEDhWXQ/uk34o=", + "ref": "refs/tags/v0.18.4", + "rev": "7a5e01071ae635e06f175233abd11d623f09cbb8", + "revCount": 2459, + "submodules": true, + "type": "git", + "url": "https://github.com/paullouisageneau/libdatachannel.git" + }, + "original": { + "ref": "refs/tags/v0.18.4", + "submodules": true, + "type": "git", + "url": "https://github.com/paullouisageneau/libdatachannel.git" + } + }, "libiec61850": { "flake": false, "locked": { @@ -52,11 +89,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1685031212, - "narHash": "sha256-lYRWOQG/YKNEeY0LPPKl6KphN+iEUWAhlXTx3oJgECA=", + "lastModified": 1686671746, + "narHash": "sha256-cPVdpUv+PzLuF6EFWPwtkT42Y6lomSA3npmW+Yzx878=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "bfa58af4da4ca3cf3f98092b1425cbb25ff7e341", + "rev": "4ab538636f64f624c49cef4393d1a0d4ebf12a66", "type": "github" }, "original": { @@ -68,7 +105,9 @@ "root": { "inputs": { "common": "common", + "fpga": "fpga", "lib60870": "lib60870", + "libdatachannel": "libdatachannel", "libiec61850": "libiec61850", "nixpkgs": "nixpkgs" } diff --git a/packaging/nix/flake.nix b/packaging/nix/flake.nix index 68532359a..b5a93cafd 100644 --- a/packaging/nix/flake.nix +++ b/packaging/nix/flake.nix @@ -5,7 +5,12 @@ nixpkgs.url = "github:NixOS/nixpkgs"; common = { - url = "github:VILLASframework/common?submodules=1"; + url = "github:VILLASframework/common"; + flake = false; + }; + + fpga = { + url = "git+https://github.com/VILLASframework/fpga.git?submodules=1"; flake = false; }; @@ -14,6 +19,14 @@ flake = false; }; + libdatachannel = { + type = "git"; + url = "https://github.com/paullouisageneau/libdatachannel.git"; + ref = "refs/tags/v0.18.4"; + submodules = true; + flake = false; + }; + libiec61850 = { url = "github:mz-automation/libiec61850/v1.5.1"; flake = false; @@ -26,68 +39,107 @@ ... } @ inputs: let inherit (nixpkgs) lib; + + # supported systems for native compilation supportedSystems = ["x86_64-linux" "aarch64-linux"]; + + # supported systems to cross compile to + supportedCrossSystems = ["aarch64-multiplatform"]; + + # generate attributes corresponding to all the supported systems forSupportedSystems = lib.genAttrs supportedSystems; - legacyPackages = forSupportedSystems ( - system: - import nixpkgs { - inherit system; - overlays = [(final: prev: self.packages.${system})]; - } - ); + + # generate attributes corresponding to all supported combinations of system and crossSystem + forSupportedCrossSystems = f: forSupportedSystems (system: lib.genAttrs supportedCrossSystems (f system)); + + # this overlay can be applied to nixpkgs (see `pkgsFor` below for an example) + overlay = final: prev: packagesWith final; + + # initialize nixpkgs for the specified `system` + pkgsFor = system: + import nixpkgs { + inherit system; + overlays = [overlay]; + }; + + # initialize nixpkgs for cross-compiling from `system` to `crossSystem` + crossPkgsFor = system: crossSystem: (pkgsFor system).pkgsCross.${crossSystem}; + + # build villas and its dependencies for the specified `pkgs` + packagesWith = pkgs: rec { + default = villas; + + villas-minimal = pkgs.callPackage ./villas.nix { + src = ../..; + version = "minimal"; + inherit (inputs) fpga common; + }; + + villas = villas-minimal.override { + version = "full"; + withAllExtras = true; + withAllFormats = true; + withAllHooks = true; + withAllNodes = true; + }; + + lib60870 = pkgs.callPackage ./lib60870.nix { + src = inputs.lib60870; + }; + + libdatachannel = pkgs.callPackage ./libdatachannel.nix { + src = inputs.libdatachannel; + }; + + libiec61850 = pkgs.callPackage ./libiec61850.nix { + src = inputs.libiec61850; + }; + }; in { - formatter = forSupportedSystems (system: legacyPackages.${system}.alejandra); + # standard flake attribute for normal packages (not cross-compiled) packages = forSupportedSystems ( - system: let - pkgs = legacyPackages.${system}; - in rec { - default = villas; - - villas-minimal = pkgs.callPackage ./villas.nix { - src = ../..; - common = inputs.common; - }; - - villas = villas-minimal.override { - withConfig = true; - withProtobuf = true; - withAllNodes = true; - }; - - lib60870 = pkgs.callPackage ./lib60870.nix { - src = inputs.lib60870; - }; - - libiec61850 = pkgs.callPackage ./libiec61850.nix { - src = inputs.libiec61850; - }; - } + system: + packagesWith (pkgsFor system) ); + + # non-standard attribute for cross-compilated packages + crossPackages = forSupportedCrossSystems ( + system: crossSystem: + packagesWith (crossPkgsFor system crossSystem) + ); + + # standard flake attribute allowing you to add the villas packages to your nixpkgs + overlays = { + default = overlay; + }; + + # standard flake attribute for defining developer environments devShells = forSupportedSystems ( system: let - pkgs = legacyPackages.${system}; - shellHook = '' - [ -z "$PS1" ] || exec $SHELL - ''; + pkgs = pkgsFor system; + shellHook = ''[ -z "$PS1" ] || exec "$SHELL"''; + hardeningDisable = ["all"]; in rec { default = full; minimal = pkgs.mkShell { - inherit shellHook; + inherit shellHook hardeningDisable; name = "minimal"; inputsFrom = [pkgs.villas-minimal]; }; full = pkgs.mkShell { - inherit shellHook; + inherit shellHook hardeningDisable; name = "full"; inputsFrom = [pkgs.villas]; }; } ); + + # standard flake attribute to add additional checks to `nix flake check` checks = forSupportedSystems ( system: let - pkgs = legacyPackages.${system}; + pkgs = pkgsFor system; in { fmt = pkgs.runCommand "check-fmt" {} '' cd ${self} @@ -95,5 +147,8 @@ ''; } ); + + # standard flake attribute specifying the formatter invoked on `nix fmt` + formatter = forSupportedSystems (system: (pkgsFor system).alejandra); }; } diff --git a/packaging/nix/libdatachannel.nix b/packaging/nix/libdatachannel.nix new file mode 100644 index 000000000..8821842e9 --- /dev/null +++ b/packaging/nix/libdatachannel.nix @@ -0,0 +1,27 @@ +{ + cmake, + lib, + libnice, + libpcap, + pkg-config, + stdenv, + src, + openssl, +}: +stdenv.mkDerivation { + pname = "libdatachannel"; + version = "villas"; + src = src; + nativeBuildInputs = [cmake pkg-config]; + buildInputs = [libnice libpcap openssl]; + cmakeFlags = [ + "-DUSE_NICE=ON" # use libnice for better protocol support + "-DNO_WEBSOCKET=ON" # villas uses libwebsockets instead + "-DNO_MEDIA=ON" # villas does not use media transport features + ]; + meta = with lib; { + description = "C/C++ WebRTC network library featuring Data Channels, Media Transport, and WebSockets"; + homepage = "https://libdatachannel.org/"; + license = licenses.mpl20; + }; +} diff --git a/packaging/nix/villas.nix b/packaging/nix/villas.nix index cf857a3e1..803df8cbb 100644 --- a/packaging/nix/villas.nix +++ b/packaging/nix/villas.nix @@ -1,72 +1,89 @@ { - # build dependencies - cmake, - common, - lib, - makeWrapper, - pkg-config, - stdenv, + # general configuration src, - # configuration - withConfig ? false, - withProtobuf ? false, + version, + withAllExtras ? false, + withAllFormats ? false, + withAllHooks ? false, withAllNodes ? false, + withExtraConfig ? withAllExtras, + withExtraGraphviz ? false, # unknown type boolean in villas-graph + withFormatProtobuf ? withAllFormats, + withHookLua ? false, # deprecated functions + withNodeAmqp ? false, # deprecated functions withNodeComedi ? withAllNodes, + withNodeFpga ? false, # spdlog formatting error withNodeIec60870 ? withAllNodes, withNodeIec61850 ? withAllNodes, withNodeInfiniband ? withAllNodes, withNodeKafka ? withAllNodes, withNodeMqtt ? withAllNodes, - withNodeRedis ? withAllNodes, - withNodeUldaq ? withAllNodes, - withNodeZeromq ? withAllNodes, withNodeNanomsg ? withAllNodes, + withNodeRedis ? false, # spdlog formatting error withNodeRtp ? withAllNodes, - withNodeTemper ? withAllNodes, withNodeSocket ? withAllNodes, - # dependencies - comedilib, + withNodeTemper ? withAllNodes, + withNodeUldaq ? withAllNodes, + withNodeWebrtc ? withAllNodes, + withNodeZeromq ? withAllNodes, + # minimal dependencies + cmake, + common, coreutils, + fpga, + graphviz, + lib, + makeWrapper, + pkg-config, + stdenv, + # optional dependencies + comedilib, curl, czmq, gnugrep, - graphviz, jansson, lib60870, libconfig, + libdatachannel, libiec61850, - libre, libnl, + libre, libsodium, libuldaq, libusb, libuuid, libwebsockets, + lua, mosquitto, nanomsg, openssl, - protobuf, + pkgsBuildBuild, protobufc, + protobufcBuildBuild ? pkgsBuildBuild.protobufc, + rabbitmq-c, rdkafka, rdma-core, + redis-plus-plus, spdlog, }: stdenv.mkDerivation { + inherit src version; pname = "villas"; - version = "release"; - src = src; - cmakeFlags = [ - "-DWITH_FPGA=OFF" - "-DDOWNLOAD_GO=OFF" - "-DCMAKE_BUILD_TYPE=Release" - # the default -O3 causes g++ warning false positives on - # 'array-bounds' and 'stringop-overflow' for villas-relay - "-DCMAKE_CXX_FLAGS_RELEASE=-Wno-error" - ]; + cmakeFlags = + [ + "-DDOWNLOAD_GO=OFF" + "-DCMAKE_BUILD_TYPE=Release" + "-DCMAKE_CXX_FLAGS_RELEASE=-Wno-error=maybe-uninitialized" + ] + ++ lib.optionals withFormatProtobuf ["-DCMAKE_FIND_ROOT_PATH=${protobufcBuildBuild}/bin"]; preConfigure = '' - rm -d common && ln -sf ${common} common + rm -df common + rm -df fpga + ln -s ${common} common + ${lib.optionalString withNodeFpga "ln -s ${fpga} fpga"} ''; postInstall = '' + patchShebangs --build $out/bin/villas wrapProgram $out/bin/villas \ --set PATH ${lib.makeBinPath [(placeholder "out") gnugrep coreutils]} ''; @@ -75,6 +92,7 @@ stdenv.mkDerivation { makeWrapper pkg-config ]; + depsBuildBuild = lib.optionals withFormatProtobuf [protobufcBuildBuild]; buildInputs = [ jansson @@ -84,21 +102,27 @@ stdenv.mkDerivation { curl spdlog ] - ++ lib.optionals withConfig [libconfig] - ++ lib.optionals withProtobuf [protobuf protobufc] + ++ lib.optionals withExtraConfig [libconfig] + ++ lib.optionals withExtraGraphviz [graphviz] + ++ lib.optionals withFormatProtobuf [protobufc] + ++ lib.optionals withHookLua [lua] + ++ lib.optionals withNodeAmqp [rabbitmq-c] ++ lib.optionals withNodeComedi [comedilib] - ++ lib.optionals withNodeZeromq [czmq libsodium] ++ lib.optionals withNodeIec60870 [lib60870] ++ lib.optionals withNodeIec61850 [libiec61850] - ++ lib.optionals withNodeSocket [libnl] - ++ lib.optionals withNodeRtp [libre] - ++ lib.optionals withNodeUldaq [libuldaq] - ++ lib.optionals withNodeTemper [libusb] + ++ lib.optionals withNodeInfiniband [rdma-core] + ++ lib.optionals withNodeKafka [rdkafka] ++ lib.optionals withNodeMqtt [mosquitto] ++ lib.optionals withNodeNanomsg [nanomsg] - ++ lib.optionals withNodeKafka [rdkafka] - ++ lib.optionals withNodeInfiniband [rdma-core]; + ++ lib.optionals withNodeRedis [redis-plus-plus] + ++ lib.optionals withNodeRtp [libre] + ++ lib.optionals withNodeSocket [libnl] + ++ lib.optionals withNodeTemper [libusb] + ++ lib.optionals withNodeUldaq [libuldaq] + ++ lib.optionals withNodeWebrtc [libdatachannel] + ++ lib.optionals withNodeZeromq [czmq libsodium]; meta = with lib; { + mainProgram = "villas"; description = "a tool connecting real-time power grid simulation equipment"; homepage = "https://villas.fein-aachen.org/"; license = licenses.asl20;