From 1fb9a89bae62f252a9098bd5e54496693d274ca4 Mon Sep 17 00:00:00 2001 From: Philipp Jungkamp Date: Mon, 20 Feb 2023 13:18:31 +0100 Subject: [PATCH] packaging-nix: initial structure - add basic flake structure - add configurable package for `villas` - add `README` tutorial as motivation - add package for `lib60870` and `libiec61850` not provided by `nixpkgs` Signed-off-by: Philipp Jungkamp --- packaging/nix/README.md | 302 ++++++++++++++++++++++++++++++++++ packaging/nix/flake.lock | 78 +++++++++ packaging/nix/flake.nix | 83 ++++++++++ packaging/nix/lib60870.nix | 18 ++ packaging/nix/libiec61850.nix | 17 ++ packaging/nix/villas.nix | 101 ++++++++++++ 6 files changed, 599 insertions(+) create mode 100644 packaging/nix/README.md create mode 100644 packaging/nix/flake.lock create mode 100644 packaging/nix/flake.nix create mode 100644 packaging/nix/lib60870.nix create mode 100644 packaging/nix/libiec61850.nix create mode 100644 packaging/nix/villas.nix diff --git a/packaging/nix/README.md b/packaging/nix/README.md new file mode 100644 index 000000000..8b103e2f7 --- /dev/null +++ b/packaging/nix/README.md @@ -0,0 +1,302 @@ + +# `villas` as a Nix Flake + +`VILLASnode` is also packaged as a Nix Flake. + +## Setup Nix + +Note that flakes are an (as of May 2023) experimental feature of the Nix +project to provide the declarative and fully reproducible builds of Nix without +the hassle of manually updating hashes. + +Using `villas` as a flake thus requires the nix package manager to be installed +and the `flakes` and `nix-command` features to be enabled. + +Documentation: +- Installation: https://nixos.org/download.html +- Enable Flakes: https://nixos.wiki/wiki/Flakes#Enable_flakes + +Check if your installation works by running e.g. +```shell +nix run nixpkgs#hello +``` + +## Getting Started + +Try to run `villas node` by typing + +```shell +# `nix run` runs an output package provided by a flake +# `github:VILLASframework/node` is the repository containing the flake +# `?dir=packaging/nix` indicates that the `flake.nix` resides in this subdirectory +# `#villas-minimal` chooses the output package you want to run +# after the `--` follow the arguments to the `villas` tool (e.g. `node -h`) +nix run 'github:VILLASframework/node?dir=packaging/nix#villas-minimal' -- node -h +``` + +The [`flake.nix`] provides 2 versions of `villas`: +- `#villas-minimal`: `villas` without most optional dependencies +- `#villas`: `villas` with most optional dependencies available to Nix + +The version chosen by default is `#villas`. Omitting the `#` suffix will +select the more complete version of `villas`. + +```shell +# omit `#` suffix in the flake reference from above +# note the difference supported nodes/formats in help output compared to above +nix run 'github:VILLASframework/node?dir=packaging/nix' -- node -h +``` + +## Simple Install + +You can also install `villas` into your local profile and have it available in +`PATH`. + +```shell +nix profile install 'github:VILLASframework/node?dir=packaging/nix#villas' +``` + +## Development + +You can easily setup a development shell environment for `villas` by using the +`devShells` provided in the [`flake.nix`] using `nix develop`. + +```shell +# create a shell with all required build dependecies but without most optional ones +nix develop ./packaging/nix#minimal + +# create a shell with all required build dependecies with most optional ones +nix develop ./packaging/nix#full + +# the default creates the `#full` shell +nix develop ./packaging/nix +``` + +## Further Reading + +- The [Nix Manual](https://nixos.org/manual/nix/stable/): + installing and using the Nix package manager and Nix language +- The [Nixpkgs Manual](https://nixos.org/manual/nixpkgs/stable/): + using the tools provided by Nixpkgs + +## Docker/OCI Images + +OCI images created using nix can be as small as the typical `FROM scratch` +images without worrying about missing dependencies or Copying things inbetween +`Dockerfile` stages. Copying exactly the application, it's dependencies an +nothing else can be done using only a few lines of Nix. + +Here we build a docker image containing the `#villas` flake output: +```shell +# `docker load` reads OCI images from stdin +# `nix build --impure --expr` builds a derivation from an impure Nix expression +# `--no-link` omits creating the `result` symlink +# `--print-out-paths` prints the built image's path to stdout +docker load < $(nix build --no-link --print-out-paths --impure --expr ' + let + ref = "github:VILLASframework/node?dir=packaging/nix"; + villas = (builtins.getFlake ref).packages.x86_64-linux.villas; + pkgs = (builtins.getFlake "nixpkgs").legacyPackages.x86_64-linux; + in + pkgs.dockerTools.buildImage { + name = "villas"; + tag = "nix"; + created = "now"; + copyToRoot = [villas]; + config.Cmd = ["${villas}/bin/villas" "node" "-h"]; + } +') +``` + +See https://nixos.org/manual/nixpkgs/stable/#sec-pkgs-dockerTools + +## Customization + +The [`villas.nix`] file contains the Nix "derivation" used to +build `villas`. Check the `# customization` options at the top to find out what +optional things can be enabled. + +### Building a customized `villas` + +Using and customizing the villas package requires some knowledge of the Nix +expression language. And the standard `override` function used extensively in +`nixpkgs`. `override` is a Nix function provided by many packages to change +dependencies or build options. All of the inputs which can be overridden can be +found at the top of [`villas.nix`]. + +See https://nixos.org/manual/nixpkgs/stable/#chap-overrides + +```shell +# `nix build` builds a derivation +# `--expr ` specifies a Nix expression which evalutes to a "derivation" +# `--impure` allows us to use a flake without specifying the exact git revision in `ref` +# `ref` is the reference to the `villas` flake we want to use +# `pkgs` is the set of `x86_64-linux` packages provided by the flake +nix build --impure --expr ' + let + ref = "github:VILLASframework/node?dir=packaging/nix"; + pkgs = (builtins.getFlake ref).packages.x86_64-linux; + in + pkgs.villas-minimal.override { + withConfig = true; + withNodeIec60870 = true; + } +' +``` + +Here we override the `#villas-minimal` package to include `libconfig` +configuration support and enable the `IEC61870-5-104` node. + +`nix build` now builds the customized `villas` and produces a `result` symlink +in the current directory. `result` now points to a directory in the Nix store +containing the installation. You can run the `villas` binary through the +symlink. + +```shell +./result/bin/villas node -h +``` + +### Making it persistent + +Making persistent build configuration changes in a canonical way involves +writing your own flake. A flake is basically a directory containing a +`flake.nix` file. Since Nix is aware of VCS, you should create your own +`flake.nix` in a new directory outside the `villas` source tree. + +Here is a basic flake to build upon: +```nix +# flake.nix +{ + description = "example of a customized villas"; + + inputs = { + villas.url = "github:VILLASframework/node?dir=packaging/nix"; + }; + + outputs = { self, villas }: + let + villas-pkgs = villas.packages.x86_64-linux; + in { + packages.x86_64-linux = rec { + default = villas-custom; + villas-custom = villas-pkgs.villas-minimal.override { + withConfig = true; + withNodeIec60870 = true; + }; + }; + }; +} +``` + +The attributes here are fairly simple: +- `description`: short description of the flake +- `inputs`: attribute set that specifies all dependencies of the flake +- `outputs`: a Nix function taking `self` and all fetched `inputs` producing + the flake outputs + +The `packages.x86_64-linux` output of a flake is a set of all packages +buildable by nix on a `x86_64-linux` system. + +You can now build/run/install your customized `villas`. `nix` commands default +to checking the current directory (and parent directories) for a `flake.nix` +and using the `default` attribute if no `#output` is specified. + +The first time you use the flake a `flake.lock` will be created pinning villas +to an exact git revision so future invocations will be exactly reproducible. + +```shell +# build custom `villas` and create the `result` symlink +nix build . + +# run custom `villas node -h` +nix run . -- node -h + +# install villas into default profile +nix profile install . + +# create a shell environment where `villas` is available in path +nix shell . + +# update the `flake.lock` if a newer `villas` is available on github +nix flake update +``` + +### Extending the flake + +Installing `villas` globally using `nix profile install` isn't really the +typical Nix usage. A more interesting use of Nix would be a custom Nix shell +environment containing your `villas-custom` and other tools of your choice. + +Here is a more complete `flake.nix` containing `devShells` available to +`nix develop` and an OCI image: +```nix +# flake.nix +{ + description = "example of a customized villas"; + + inputs = { + # nixpkgs from the `unstable` branch + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + + # villas from the official repository + villas.url = "github:VILLASframework/node?dir=packaging/nix"; + + # overwrite the `nixpkgs` used by `villas` to our version above + # + # this will negate the reproducibility of the `villas` flake, + # but allows us to deduplicate the dependencies used in our shell + # and those in villas. + # + # comment this line if an updated dependency causes a build failure + villas.inputs.nixpkgs.follows = "nixpkgs"; + }; + + outputs = { self, villas }: + let + villas-pkgs = villas.packages.x86_64-linux; + pkgs = nixpkgs.packages.x86_64-linux; + in { + packages.x86_64-linux = rec { + # run/build/install the `villas-custom` package by default + default = villas-custom; + + # the customized villas package + villas-custom = villas-pkgs.villas-minimal.override { + withConfig = true; + withNodeIec60870 = true; + }; + + # an OCI image containing `villas-custom` + image = pkgs.dockerTools.buildImage { + name = "villas"; + tag = "nix"; + created = "now"; + copyToRoot = [villas-custom]; + config.Cmd = ["${villas-custom}/bin/villas" "node" "-h"]; + }; + }; + devShells.x86_64-linux.default = pkgs.mkShell { + name = "my-villas-dev-shell"; + shellHook = "exec $SHELL"; + packages = with pkgs; [ + self.packages.x86_64-linux.villas-custom + jq + bat + ]; + } + }; +} +``` + +You can use our new features like this: + +```shell +# run your shell for villas-node +nix develop + +# load your custom OCI image into docker +docker load < $(nix build --no-link --print-out-paths .#image) +``` + +[`villas.nix`]: ./villas.nix +[`flake.nix`]: ./flake.nix diff --git a/packaging/nix/flake.lock b/packaging/nix/flake.lock new file mode 100644 index 000000000..6a4bf9877 --- /dev/null +++ b/packaging/nix/flake.lock @@ -0,0 +1,78 @@ +{ + "nodes": { + "lib60870": { + "flake": false, + "locked": { + "lastModified": 1672404819, + "narHash": "sha256-9o+gWQbpCJb+UZzPNmzGqpWD0QbGjg41is/f1POUEQs=", + "owner": "mz-automation", + "repo": "lib60870", + "rev": "53a6b3c1cf3023e51cf81763b1ccf048edcd1c64", + "type": "github" + }, + "original": { + "owner": "mz-automation", + "ref": "v2.3.2", + "repo": "lib60870", + "type": "github" + } + }, + "libiec61850": { + "flake": false, + "locked": { + "lastModified": 1647022552, + "narHash": "sha256-1vT0ry6IJqilpM7g9l7fx+ET+Dyo24WAyWqTyPM9nQw=", + "owner": "mz-automation", + "repo": "libiec61850", + "rev": "210cf30897631fe2006ac50483caf8fd616622a2", + "type": "github" + }, + "original": { + "owner": "mz-automation", + "ref": "v1.5.1", + "repo": "libiec61850", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1683023950, + "narHash": "sha256-wG7aeu8o+tbf8VaQMDZYQ3kG3nYjklyKjUENVWPrcYY=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "826418fc35e4cd6e85a88edd389cb249a7aa9671", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "lib60870": "lib60870", + "libiec61850": "libiec61850", + "nixpkgs": "nixpkgs", + "src": "src" + } + }, + "src": { + "flake": false, + "locked": { + "lastModified": 1683025627, + "narHash": "sha256-22uKohl9nRAkhPgnOjnXzqOR1FWMpcDNSSNbkUHud5Y=", + "submodules": true, + "type": "git", + "url": "file:." + }, + "original": { + "submodules": true, + "type": "git", + "url": "file:." + } + } + }, + "root": "root", + "version": 7 +} diff --git a/packaging/nix/flake.nix b/packaging/nix/flake.nix new file mode 100644 index 000000000..ab1102ae5 --- /dev/null +++ b/packaging/nix/flake.nix @@ -0,0 +1,83 @@ +{ + description = "a tool for connecting real-time power grid simulation equipment"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs"; + + src = { + url = "git+file:.?submodules=1"; + flake = false; + }; + + lib60870 = { + url = "github:mz-automation/lib60870/v2.3.2"; + flake = false; + }; + + libiec61850 = { + url = "github:mz-automation/libiec61850/v1.5.1"; + flake = false; + }; + }; + + outputs = { + self, + nixpkgs, + ... + } @ inputs: let + inherit (nixpkgs) lib; + supportedSystems = ["x86_64-linux" "aarch64-linux"]; + forSupportedSystems = lib.genAttrs supportedSystems; + legacyPackages = forSupportedSystems ( + system: + import nixpkgs { + inherit system; + overlays = [(final: prev: self.packages.${system})]; + } + ); + in { + formatter = forSupportedSystems (system: legacyPackages.${system}.alejandra); + packages = forSupportedSystems (system: + let pkgs = legacyPackages.${system}; + in rec { + default = villas; + + villas-minimal = pkgs.callPackage ./villas.nix { + inherit (inputs) src; + }; + + villas = villas-minimal.override { + withConfig = true; + withProtobuf = true; + withAllNodes = true; + }; + + lib60870 = pkgs.callPackage ./lib60870.nix { + src = inputs.lib60870; + }; + + libiec61850 = pkgs.callPackage ./libiec61850.nix { + src = inputs.libiec61850; + }; + } + ); + devShells = forSupportedSystems (system: + let pkgs = legacyPackages.${system}; + in rec { + default = full; + + minimal = pkgs.mkShell { + name = "minimal"; + shellHook = "exec $SHELL"; + inputsFrom = [ pkgs.villas-minimal ]; + }; + + full = pkgs.mkShell { + name = "full"; + shellHook = "exec $SHELL"; + inputsFrom = [ pkgs.villas ]; + }; + } + ); + }; +} diff --git a/packaging/nix/lib60870.nix b/packaging/nix/lib60870.nix new file mode 100644 index 000000000..aadef1e4c --- /dev/null +++ b/packaging/nix/lib60870.nix @@ -0,0 +1,18 @@ +{ + cmake, + lib, + stdenv, + src +}: +stdenv.mkDerivation { + pname = "lib60870"; + version = "villas"; + src = src; + nativeBuildInputs = [cmake]; + preConfigure = "cd lib60870-C"; + meta = with lib; { + description = "implementation of the IEC 60870-5-101/104 protocol"; + homepage = "https://libiec61850.com/"; + license = licenses.gpl3; + }; +} diff --git a/packaging/nix/libiec61850.nix b/packaging/nix/libiec61850.nix new file mode 100644 index 000000000..ebb2a6bf3 --- /dev/null +++ b/packaging/nix/libiec61850.nix @@ -0,0 +1,17 @@ +{ + cmake, + lib, + stdenv, + src +}: +stdenv.mkDerivation { + pname = "libiec61850"; + version = "villas"; + src = src; + nativeBuildInputs = [cmake]; + meta = with lib; { + description = "open-source library for the IEC 61850 protocols"; + homepage = "https://libiec61850.com/"; + license = licenses.gpl3; + }; +} diff --git a/packaging/nix/villas.nix b/packaging/nix/villas.nix new file mode 100644 index 000000000..ccf2ab65d --- /dev/null +++ b/packaging/nix/villas.nix @@ -0,0 +1,101 @@ +{ + # build dependencies + cmake, + lib, + makeWrapper, + pkg-config, + stdenv, + src, + # configuration + withConfig ? false, + withProtobuf ? false, + withAllNodes ? false, + withNodeComedi ? withAllNodes, + withNodeIec60870 ? withAllNodes, + withNodeIec61850 ? withAllNodes, + withNodeInfiniband ? withAllNodes, + withNodeKafka ? withAllNodes, + withNodeMqtt ? withAllNodes, + withNodeRedis ? withAllNodes, + withNodeUldaq ? withAllNodes, + withNodeZeromq ? withAllNodes, + withNodeNanomsg ? withAllNodes, + withNodeRtp ? withAllNodes, + withNodeTemper ? withAllNodes, + withNodeSocket ? withAllNodes, + # dependencies + comedilib, + curl, + czmq, + gnugrep, + graphviz, + jansson, + lib60870, + libconfig, + libiec61850, + libre, + libnl, + libsodium, + libuldaq, + libusb, + libuuid, + libwebsockets, + mosquitto, + nanomsg, + openssl, + protobuf, + protobufc, + rdkafka, + rdma-core, + spdlog +}: +stdenv.mkDerivation { + pname = "villas"; + version = "release"; + src = src; + cmakeFlags = [ + "-DWITH_FPGA=OFF" + "-DDOWNLOAD_GO=OFF" + "-DCMAKE_BUILD_TYPE=Release" + # the default -O3 causes g++ warning false positives on + # 'array-bounds' and 'stringop-overflow' for villas-relay + "-DCMAKE_CXX_FLAGS_RELEASE=-Wno-error" + ]; + postInstall = '' + wrapProgram $out/bin/villas \ + --set PATH ${lib.makeBinPath [(placeholder "out") gnugrep]} + ''; + nativeBuildInputs = [ + cmake + makeWrapper + pkg-config + ]; + buildInputs = [ + jansson + libwebsockets + libuuid + openssl + curl + spdlog + ] + ++ lib.optionals withConfig [libconfig] + ++ lib.optionals withProtobuf [protobuf protobufc] + ++ lib.optionals withNodeComedi [comedilib] + ++ lib.optionals withNodeZeromq [czmq libsodium] + ++ lib.optionals withNodeIec60870 [lib60870] + ++ lib.optionals withNodeIec61850 [libiec61850] + ++ lib.optionals withNodeSocket [libnl] + ++ lib.optionals withNodeRtp [libre] + ++ lib.optionals withNodeUldaq [libuldaq] + ++ lib.optionals withNodeTemper [libusb] + ++ lib.optionals withNodeMqtt [mosquitto] + ++ lib.optionals withNodeNanomsg [nanomsg] + ++ lib.optionals withNodeKafka [rdkafka] + ++ lib.optionals withNodeInfiniband [rdma-core] + ; + meta = with lib; { + description = "a tool connecting real-time power grid simulation equipment"; + homepage = "https://villas.fein-aachen.org/"; + license = licenses.asl20; + }; +}