diff --git a/nixos/modules/virtualisation/lxd.nix b/nixos/modules/virtualisation/lxd.nix index 53b89a9f55b1..3958fc2c1d7c 100644 --- a/nixos/modules/virtualisation/lxd.nix +++ b/nixos/modules/virtualisation/lxd.nix @@ -15,7 +15,6 @@ in ###### interface options = { - virtualisation.lxd = { enable = mkOption { type = types.bool; @@ -25,12 +24,18 @@ in containers. Users in the "lxd" group can interact with the daemon (e.g. to start or stop containers) using the lxc command line tool, among others. + + Most of the time, you'll also want to start lxcfs, so + that containers can "see" the limits: + + virtualisation.lxc.lxcfs.enable = true; + ''; }; package = mkOption { type = types.package; - default = pkgs.lxd; + default = pkgs.lxd.override { nftablesSupport = config.networking.nftables.enable; }; defaultText = "pkgs.lxd"; description = '' The LXD package to use. @@ -65,6 +70,7 @@ in with nixos. ''; }; + recommendedSysctlSettings = mkOption { type = types.bool; default = false; @@ -83,7 +89,6 @@ in ###### implementation config = mkIf cfg.enable { - environment.systemPackages = [ cfg.package ]; security.apparmor = { @@ -115,6 +120,12 @@ in LimitNOFILE = "1048576"; LimitNPROC = "infinity"; TasksMax = "infinity"; + + # By default, `lxd` loads configuration files from hard-coded + # `/usr/share/lxc/config` - since this is a no-go for us, we have to + # explicitly tell it where the actual configuration files are + Environment = mkIf (config.virtualisation.lxc.lxcfs.enable) + "LXD_LXC_TEMPLATE_CONFIG=${pkgs.lxcfs}/share/lxc/config"; }; }; diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 8e262d8eee7d..1bf091b361cb 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -178,6 +178,8 @@ in limesurvey = handleTest ./limesurvey.nix {}; login = handleTest ./login.nix {}; loki = handleTest ./loki.nix {}; + lxd = handleTest ./lxd.nix {}; + lxd-nftables = handleTest ./lxd-nftables.nix {}; #logstash = handleTest ./logstash.nix {}; lorri = handleTest ./lorri/default.nix {}; magnetico = handleTest ./magnetico.nix {}; diff --git a/nixos/tests/lxd-nftables.nix b/nixos/tests/lxd-nftables.nix new file mode 100644 index 000000000000..25517914db85 --- /dev/null +++ b/nixos/tests/lxd-nftables.nix @@ -0,0 +1,50 @@ +# This test makes sure that lxd stops implicitly depending on iptables when +# user enabled nftables. +# +# It has been extracted from `lxd.nix` for clarity, and because switching from +# iptables to nftables requires a full reboot, which is a bit hard inside NixOS +# tests. + +import ./make-test-python.nix ({ pkgs, ...} : { + name = "lxd-nftables"; + meta = with pkgs.stdenv.lib.maintainers; { + maintainers = [ patryk27 ]; + }; + + machine = { lib, ... }: { + virtualisation = { + lxd.enable = true; + }; + + networking = { + firewall.enable = false; + nftables.enable = true; + nftables.ruleset = '' + table inet filter { + chain incoming { + type filter hook input priority 0; + policy accept; + } + + chain forward { + type filter hook forward priority 0; + policy accept; + } + + chain output { + type filter hook output priority 0; + policy accept; + } + } + ''; + }; + }; + + testScript = '' + machine.wait_for_unit("network.target") + + with subtest("When nftables are enabled, lxd doesn't depend on iptables anymore"): + machine.succeed("lsmod | grep nf_tables") + machine.fail("lsmod | grep ip_tables") + ''; +}) diff --git a/nixos/tests/lxd.nix b/nixos/tests/lxd.nix new file mode 100644 index 000000000000..db2d44dff557 --- /dev/null +++ b/nixos/tests/lxd.nix @@ -0,0 +1,135 @@ +import ./make-test-python.nix ({ pkgs, ...} : + +let + # Since we don't have access to the internet during the tests, we have to + # pre-fetch lxd containers beforehand. + # + # I've chosen to import Alpine Linux, because its image is turbo-tiny and, + # generally, sufficient for our tests. + + alpine-meta = pkgs.fetchurl { + url = "https://uk.images.linuxcontainers.org/images/alpine/3.11/i386/default/20200608_13:00/lxd.tar.xz"; + sha256 = "1hkvaj3rr333zmx1759njy435lps33gl4ks8zfm7m4nqvipm26a0"; + }; + + alpine-rootfs = pkgs.fetchurl { + url = "https://uk.images.linuxcontainers.org/images/alpine/3.11/i386/default/20200608_13:00/rootfs.tar.xz"; + sha256 = "1v82zdra4j5xwsff09qlp7h5vbsg54s0j7rdg4rynichfid3r347"; + }; + + lxd-config = pkgs.writeText "config.yaml" '' + storage_pools: + - name: default + driver: dir + config: + source: /var/lxd-pool + + networks: + - name: lxdbr0 + type: bridge + config: + ipv4.address: auto + ipv6.address: none + + profiles: + - name: default + devices: + eth0: + name: eth0 + network: lxdbr0 + type: nic + root: + path: / + pool: default + type: disk + ''; + +in { + name = "lxd"; + meta = with pkgs.stdenv.lib.maintainers; { + maintainers = [ patryk27 ]; + }; + + machine = { lib, ... }: { + virtualisation = { + # Since we're testing `limits.cpu`, we've gotta have a known number of + # cores to lay on + cores = 2; + + # Ditto, for `limits.memory` + memorySize = 512; + + lxc.lxcfs.enable = true; + lxd.enable = true; + }; + }; + + testScript = '' + machine.wait_for_unit("sockets.target") + machine.wait_for_unit("lxd.service") + + # It takes additional second for lxd to settle + machine.sleep(1) + + # lxd expects the pool's directory to already exist + machine.succeed("mkdir /var/lxd-pool") + + machine.succeed( + "cat ${lxd-config} | lxd init --preseed" + ) + + machine.succeed( + "lxc image import ${alpine-meta} ${alpine-rootfs} --alias alpine" + ) + + with subtest("Containers can be launched and destroyed"): + machine.succeed("lxc launch alpine test") + machine.succeed("lxc exec test true") + machine.succeed("lxc delete -f test") + + with subtest("Containers are being mounted with lxcfs inside"): + machine.succeed("lxc launch alpine test") + + ## ---------- ## + ## limits.cpu ## + + machine.succeed("lxc config set test limits.cpu 1") + + # Since Alpine doesn't have `nproc` pre-installed, we've gotta resort + # to the primal methods + assert ( + "1" + == machine.succeed("lxc exec test grep -- -c ^processor /proc/cpuinfo").strip() + ) + + machine.succeed("lxc config set test limits.cpu 2") + + assert ( + "2" + == machine.succeed("lxc exec test grep -- -c ^processor /proc/cpuinfo").strip() + ) + + ## ------------- ## + ## limits.memory ## + + machine.succeed("lxc config set test limits.memory 64MB") + + assert ( + "MemTotal: 62500 kB" + == machine.succeed("lxc exec test grep -- MemTotal /proc/meminfo").strip() + ) + + machine.succeed("lxc config set test limits.memory 128MB") + + assert ( + "MemTotal: 125000 kB" + == machine.succeed("lxc exec test grep -- MemTotal /proc/meminfo").strip() + ) + + machine.succeed("lxc delete -f test") + + with subtest("Unless explicitly changed, lxd leans on iptables"): + machine.succeed("lsmod | grep ip_tables") + machine.fail("lsmod | grep nf_tables") + ''; +}) diff --git a/pkgs/os-specific/linux/lxcfs/default.nix b/pkgs/os-specific/linux/lxcfs/default.nix index 68d05f0be65d..4011b3885625 100644 --- a/pkgs/os-specific/linux/lxcfs/default.nix +++ b/pkgs/os-specific/linux/lxcfs/default.nix @@ -1,4 +1,5 @@ { config, stdenv, fetchFromGitHub, autoreconfHook, pkgconfig, help2man, fuse +, utillinux, makeWrapper , enableDebugBuild ? config.lxcfs.enableDebugBuild or false }: with stdenv.lib; @@ -13,7 +14,7 @@ stdenv.mkDerivation rec { }; nativeBuildInputs = [ pkgconfig help2man autoreconfHook ]; - buildInputs = [ fuse ]; + buildInputs = [ fuse makeWrapper ]; preConfigure = stdenv.lib.optionalString enableDebugBuild '' sed -i 's,#AM_CFLAGS += -DDEBUG,AM_CFLAGS += -DDEBUG,' Makefile.am @@ -27,6 +28,12 @@ stdenv.mkDerivation rec { installFlags = [ "SYSTEMD_UNIT_DIR=\${out}/lib/systemd" ]; + postInstall = '' + # `mount` hook requires access to the `mount` command from `utillinux`: + wrapProgram "$out/share/lxcfs/lxc.mount.hook" \ + --prefix PATH : "${utillinux}/bin" + ''; + postFixup = '' # liblxcfs.so is reloaded with dlopen() patchelf --set-rpath "$(patchelf --print-rpath "$out/bin/lxcfs"):$out/lib" "$out/bin/lxcfs" diff --git a/pkgs/tools/admin/lxd/default.nix b/pkgs/tools/admin/lxd/default.nix index 43cd243561b5..562149414b3d 100644 --- a/pkgs/tools/admin/lxd/default.nix +++ b/pkgs/tools/admin/lxd/default.nix @@ -1,13 +1,21 @@ { stdenv, hwdata, pkgconfig, lxc, buildGoPackage, fetchurl , makeWrapper, acl, rsync, gnutar, xz, btrfs-progs, gzip, dnsmasq -, squashfsTools, iproute, iptables, ebtables, libcap, libco-canonical, dqlite -, raft-canonical, sqlite-replication, udev +, squashfsTools, iproute, iptables, ebtables, iptables-nftables-compat, libcap +, libco-canonical, dqlite, raft-canonical, sqlite-replication, udev , writeShellScriptBin, apparmor-profiles, apparmor-parser , criu , bash , installShellFiles +, nftablesSupport ? false }: +let + networkPkgs = if nftablesSupport then + [ iptables-nftables-compat ] + else + [ iptables ebtables ]; + +in buildGoPackage rec { pname = "lxd"; version = "4.2"; @@ -38,12 +46,14 @@ buildGoPackage rec { # test binaries, code generation rm $out/bin/{deps,macaroon-identity,generate} - wrapProgram $out/bin/lxd --prefix PATH : ${stdenv.lib.makeBinPath [ - acl rsync gnutar xz btrfs-progs gzip dnsmasq squashfsTools iproute iptables ebtables bash criu - (writeShellScriptBin "apparmor_parser" '' - exec '${apparmor-parser}/bin/apparmor_parser' -I '${apparmor-profiles}/etc/apparmor.d' "$@" - '') - ]} + wrapProgram $out/bin/lxd --prefix PATH : ${stdenv.lib.makeBinPath ( + networkPkgs + ++ [ acl rsync gnutar xz btrfs-progs gzip dnsmasq squashfsTools iproute bash criu ] + ++ [ (writeShellScriptBin "apparmor_parser" '' + exec '${apparmor-parser}/bin/apparmor_parser' -I '${apparmor-profiles}/etc/apparmor.d' "$@" + '') ] + ) + } installShellCompletion --bash go/src/github.com/lxc/lxd/scripts/bash/lxd-client '';