From bc138f407f9cc1206dd13aa233f2d870936e10bd Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Fri, 17 Apr 2020 07:39:54 -0700 Subject: [PATCH] nixos/k3s: add initial k3s service * nixos/k3s: simplify config expression * nixos/k3s: add config assertions and trim unneeded bits * nixos/k3s: add a test that k3s works; minor module improvements This is a single-node test. Eventually we should also have a multi-node test to verify the agent bit works, but that one's more involved. * nixos/k3s: add option description * nixos/k3s: add defaults for token/serveraddr Now that the assertion enforces their presence, we dont' need to use the typesystem for it. * nixos/k3s: remove unneeded sudo in test * nixos/k3s: add to test list --- nixos/modules/module-list.nix | 1 + .../modules/services/cluster/k3s/default.nix | 101 ++++++++++++++++++ nixos/tests/all-tests.nix | 1 + nixos/tests/k3s.nix | 78 ++++++++++++++ 4 files changed, 181 insertions(+) create mode 100644 nixos/modules/services/cluster/k3s/default.nix create mode 100644 nixos/tests/k3s.nix diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index c3d2bb85809e..41304988a596 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -238,6 +238,7 @@ ./services/backup/zfs-replication.nix ./services/backup/znapzend.nix ./services/cluster/hadoop/default.nix + ./services/cluster/k3s/default.nix ./services/cluster/kubernetes/addons/dns.nix ./services/cluster/kubernetes/addons/dashboard.nix ./services/cluster/kubernetes/addon-manager.nix diff --git a/nixos/modules/services/cluster/k3s/default.nix b/nixos/modules/services/cluster/k3s/default.nix new file mode 100644 index 000000000000..ccdf54d128e2 --- /dev/null +++ b/nixos/modules/services/cluster/k3s/default.nix @@ -0,0 +1,101 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + cfg = config.services.k3s; +in +{ + # interface + options.services.k3s = { + enable = mkEnableOption "k3s"; + + package = mkOption { + type = types.package; + default = pkgs.k3s; + defaultText = "pkgs.k3s"; + example = literalExample "pkgs.k3s"; + description = "Package that should be used for k3s"; + }; + + role = mkOption { + description = '' + Whether k3s should run as a server or agent. + Note that the server, by default, also runs as an agent. + ''; + default = "server"; + type = types.enum [ "server" "agent" ]; + }; + + serverAddr = mkOption { + type = types.str; + description = "The k3s server to connect to. This option only makes sense for an agent."; + example = "https://10.0.0.10:6443"; + default = ""; + }; + + token = mkOption { + type = types.str; + description = "The k3s token to use when connecting to the server. This option only makes sense for an agent."; + default = ""; + }; + + docker = mkOption { + type = types.bool; + default = false; + description = "Use docker to run containers rather than the built-in containerd."; + }; + + extraFlags = mkOption { + description = "Extra flags to pass to the k3s command."; + default = ""; + example = "--no-deploy traefik --cluster-cidr 10.24.0.0/16"; + }; + + disableAgent = mkOption { + type = types.bool; + default = false; + description = "Only run the server. This option only makes sense for a server."; + }; + }; + + # implementation + + config = mkIf cfg.enable { + assertions = [ + { + assertion = cfg.role == "agent" -> cfg.serverAddr != ""; + message = "serverAddr should be set if role is 'agent'"; + } + { + assertion = cfg.role == "agent" -> cfg.token != ""; + message = "token should be set if role is 'agent'"; + } + ]; + + virtualisation.docker = mkIf cfg.docker { + enable = mkDefault true; + }; + + systemd.services.k3s = { + description = "k3s service"; + after = mkIf cfg.docker [ "docker.service" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + # Taken from https://github.com/rancher/k3s/blob/v1.17.4+k3s1/contrib/ansible/roles/k3s/node/templates/k3s.service.j2 + Type = "notify"; + KillMode = "process"; + Delegate = "yes"; + Restart = "always"; + RestartSec = "5s"; + ExecStart = concatStringsSep " \\\n " ( + [ + "${cfg.package}/bin/k3s ${cfg.role}" + ] ++ (optional cfg.docker "--docker") + ++ (optional cfg.disableAgent "--disable-agent") + ++ (optional (cfg.role == "agent") "--server ${cfg.serverAddr} --token ${cfg.token}") + ++ [ cfg.extraFlags ] + ); + }; + }; + }; +} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 4f60892488ab..5adfd6afd6a7 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -146,6 +146,7 @@ in jellyfin = handleTest ./jellyfin.nix {}; jenkins = handleTest ./jenkins.nix {}; jirafeau = handleTest ./jirafeau.nix {}; + k3s = handleTest ./k3s.nix {}; kafka = handleTest ./kafka.nix {}; keepalived = handleTest ./keepalived.nix {}; kerberos = handleTest ./kerberos/default.nix {}; diff --git a/nixos/tests/k3s.nix b/nixos/tests/k3s.nix new file mode 100644 index 000000000000..5bda6f493f0e --- /dev/null +++ b/nixos/tests/k3s.nix @@ -0,0 +1,78 @@ +import ./make-test-python.nix ({ pkgs, ... }: + +let + # A suitable k3s pause image, also used for the test pod + pauseImage = pkgs.dockerTools.buildImage { + name = "test.local/pause"; + tag = "local"; + contents = with pkgs; [ tini coreutils busybox ]; + config.Entrypoint = [ "/bin/tini" "--" "/bin/sleep" "inf" ]; + }; + testPodYaml = pkgs.writeText "test.yml" '' + # Don't use the default service account because there's a race where it may + # not be created yet; make our own instead. + apiVersion: v1 + kind: ServiceAccount + metadata: + name: test + --- + apiVersion: v1 + kind: Pod + metadata: + name: test + spec: + serviceAccountName: test + containers: + - name: test + image: test.local/pause:local + imagePullPolicy: Never + command: ["sh", "-c", "sleep inf"] + ''; +in +{ + name = "k3s"; + meta = with pkgs.stdenv.lib.maintainers; { + maintainers = [ euank ]; + }; + + nodes = { + k3s = + { pkgs, ... }: { + environment.systemPackages = [ pkgs.k3s pkgs.gzip ]; + + # k3s uses enough resources the default vm fails. + virtualisation.memorySize = pkgs.lib.mkDefault 1536; + virtualisation.diskSize = pkgs.lib.mkDefault 4096; + + services.k3s.enable = true; + services.k3s.role = "server"; + services.k3s.package = pkgs.k3s; + # Slightly reduce resource usage + services.k3s.extraFlags = "--no-deploy coredns,servicelb,traefik,local-storage,metrics-server --pause-image test.local/pause:local"; + + users.users = { + noprivs = { + isNormalUser = true; + description = "Can't access k3s by default"; + password = "*"; + }; + }; + }; + }; + + testScript = '' + start_all() + + k3s.wait_for_unit("k3s") + k3s.succeed("k3s kubectl cluster-info") + k3s.fail("sudo -u noprivs k3s kubectl cluster-info") + # k3s.succeed("k3s check-config") # fails with the current nixos kernel config, uncomment once this passes + + k3s.succeed( + "zcat ${pauseImage} | k3s ctr image import -" + ) + + k3s.succeed("k3s kubectl apply -f ${testPodYaml}") + k3s.succeed("k3s kubectl wait --for 'condition=Ready' pod/test") + ''; +})