From 3210ce7784cb462659e7b0d29db8c40bd7b7e99f Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Sat, 23 Oct 2021 18:05:32 -0700 Subject: [PATCH 1/5] nixos/tests/k3s: cleanly shutdown on test success When this test was written, I don't think the explicit '.shutdown' was required, but it is now. Do so. --- nixos/tests/k3s.nix | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nixos/tests/k3s.nix b/nixos/tests/k3s.nix index 494a3b68b59d..3d1584fa2e2a 100644 --- a/nixos/tests/k3s.nix +++ b/nixos/tests/k3s.nix @@ -8,9 +8,9 @@ let contents = with pkgs; [ tini coreutils busybox ]; config.Entrypoint = [ "/bin/tini" "--" "/bin/sleep" "inf" ]; }; + # Don't use the default service account because there's a race where it may + # not be created yet; make our own instead. testPodYaml = pkgs.writeText "test.yml" '' - # Don't use the default service account because there's a race where it may - # not be created yet; make our own instead. apiVersion: v1 kind: ServiceAccount metadata: @@ -38,7 +38,7 @@ in nodes = { k3s = { pkgs, ... }: { - environment.systemPackages = [ pkgs.k3s pkgs.gzip ]; + environment.systemPackages = with pkgs; [ k3s gzip ]; # k3s uses enough resources the default vm fails. virtualisation.memorySize = pkgs.lib.mkDefault 1536; @@ -74,5 +74,8 @@ in k3s.succeed("k3s kubectl apply -f ${testPodYaml}") k3s.succeed("k3s kubectl wait --for 'condition=Ready' pod/test") + k3s.succeed("k3s kubectl delete -f ${testPodYaml}") + + k3s.shutdown() ''; }) From f20af9dbfbe54dfb1ba557fcc7edb316559a9130 Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Sat, 23 Oct 2021 18:21:47 -0700 Subject: [PATCH 2/5] nixos/tests/k3s: make more idiomatic For single-node tests, using machine is more idiomatic from what I gather, so do that. I do want multi-node tests, but those should be in different files. --- nixos/tests/k3s.nix | 53 +++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/nixos/tests/k3s.nix b/nixos/tests/k3s.nix index 3d1584fa2e2a..ff329af3c5ec 100644 --- a/nixos/tests/k3s.nix +++ b/nixos/tests/k3s.nix @@ -35,47 +35,44 @@ in maintainers = [ euank ]; }; - nodes = { - k3s = - { pkgs, ... }: { - environment.systemPackages = with pkgs; [ k3s gzip ]; + machine = { pkgs, ... }: { + environment.systemPackages = with pkgs; [ k3s gzip ]; - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = pkgs.lib.mkDefault 1536; - virtualisation.diskSize = pkgs.lib.mkDefault 4096; + # k3s uses enough resources the default vm fails. + virtualisation.memorySize = pkgs.lib.mkDefault 1536; + virtualisation.diskSize = pkgs.lib.mkDefault 4096; - services.k3s.enable = true; - services.k3s.role = "server"; - services.k3s.package = pkgs.k3s; - # Slightly reduce resource usage - services.k3s.extraFlags = "--no-deploy coredns,servicelb,traefik,local-storage,metrics-server --pause-image test.local/pause:local"; + services.k3s.enable = true; + services.k3s.role = "server"; + services.k3s.package = pkgs.k3s; + # Slightly reduce resource usage + services.k3s.extraFlags = "--no-deploy coredns,servicelb,traefik,local-storage,metrics-server --pause-image test.local/pause:local"; - users.users = { - noprivs = { - isNormalUser = true; - description = "Can't access k3s by default"; - password = "*"; - }; - }; + users.users = { + noprivs = { + isNormalUser = true; + description = "Can't access k3s by default"; + password = "*"; }; + }; }; testScript = '' start_all() - k3s.wait_for_unit("k3s") - k3s.succeed("k3s kubectl cluster-info") - k3s.fail("sudo -u noprivs k3s kubectl cluster-info") - # k3s.succeed("k3s check-config") # fails with the current nixos kernel config, uncomment once this passes + machine.wait_for_unit("k3s") + machine.succeed("k3s kubectl cluster-info") + machine.fail("sudo -u noprivs k3s kubectl cluster-info") + # machine.succeed("k3s check-config") # fails with the current nixos kernel config, uncomment once this passes - k3s.succeed( + machine.succeed( "zcat ${pauseImage} | k3s ctr image import -" ) - k3s.succeed("k3s kubectl apply -f ${testPodYaml}") - k3s.succeed("k3s kubectl wait --for 'condition=Ready' pod/test") - k3s.succeed("k3s kubectl delete -f ${testPodYaml}") + machine.succeed("k3s kubectl apply -f ${testPodYaml}") + machine.succeed("k3s kubectl wait --for 'condition=Ready' pod/test") + machine.succeed("k3s kubectl delete -f ${testPodYaml}") - k3s.shutdown() + machine.shutdown() ''; }) From 8bf9500e6576c7f6b150834ab6e55409d056979c Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Sat, 23 Oct 2021 18:29:05 -0700 Subject: [PATCH 3/5] nixos/tests/k3s-docker: add 'k3s.docker=true' test This verifies the docker driver works too, which matters because it interacts with systemd, cgroups, etc differently. --- nixos/tests/k3s-docker.nix | 80 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 nixos/tests/k3s-docker.nix diff --git a/nixos/tests/k3s-docker.nix b/nixos/tests/k3s-docker.nix new file mode 100644 index 000000000000..d6575bd6fff4 --- /dev/null +++ b/nixos/tests/k3s-docker.nix @@ -0,0 +1,80 @@ +import ./make-test-python.nix ({ pkgs, ... }: + +let + # A suitable k3s pause image, also used for the test pod + pauseImage = pkgs.dockerTools.buildImage { + name = "test.local/pause"; + tag = "local"; + contents = with pkgs; [ tini coreutils busybox ]; + config.Entrypoint = [ "/bin/tini" "--" "/bin/sleep" "inf" ]; + }; + # Don't use the default service account because there's a race where it may + # not be created yet; make our own instead. + testPodYaml = pkgs.writeText "test.yml" '' + apiVersion: v1 + kind: ServiceAccount + metadata: + name: test + --- + apiVersion: v1 + kind: Pod + metadata: + name: test + spec: + serviceAccountName: test + containers: + - name: test + image: test.local/pause:local + imagePullPolicy: Never + command: ["sh", "-c", "sleep inf"] + ''; +in +{ + name = "k3s"; + meta = with pkgs.lib.maintainers; { + maintainers = [ euank ]; + }; + + machine = { pkgs, ... }: { + environment.systemPackages = with pkgs; [ k3s gzip ]; + + # k3s uses enough resources the default vm fails. + virtualisation.memorySize = pkgs.lib.mkDefault 1536; + virtualisation.diskSize = pkgs.lib.mkDefault 4096; + + services.k3s = { + enable = true; + role = "server"; + docker = true; + # Slightly reduce resource usage + extraFlags = "--no-deploy coredns,servicelb,traefik,local-storage,metrics-server --pause-image test.local/pause:local"; + }; + + users.users = { + noprivs = { + isNormalUser = true; + description = "Can't access k3s by default"; + password = "*"; + }; + }; + }; + + testScript = '' + start_all() + + machine.wait_for_unit("k3s") + machine.succeed("k3s kubectl cluster-info") + machine.fail("sudo -u noprivs k3s kubectl cluster-info") + # machine.succeed("k3s check-config") # fails with the current nixos kernel config, uncomment once this passes + + machine.succeed( + "zcat ${pauseImage} | docker load" + ) + + machine.succeed("k3s kubectl apply -f ${testPodYaml}") + machine.succeed("k3s kubectl wait --for 'condition=Ready' pod/test") + machine.succeed("k3s kubectl delete -f ${testPodYaml}") + + machine.shutdown() + ''; +}) From 83ed4b46fde24845c3157dcf0923210d6423998c Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Sat, 23 Oct 2021 18:34:10 -0700 Subject: [PATCH 4/5] nixos/tests/k3s: nixpkgs-fmt --- nixos/tests/k3s-docker.nix | 138 ++++++++++++++++++------------------- nixos/tests/k3s.nix | 130 +++++++++++++++++----------------- 2 files changed, 134 insertions(+), 134 deletions(-) diff --git a/nixos/tests/k3s-docker.nix b/nixos/tests/k3s-docker.nix index d6575bd6fff4..62544e3ee13e 100644 --- a/nixos/tests/k3s-docker.nix +++ b/nixos/tests/k3s-docker.nix @@ -1,80 +1,80 @@ import ./make-test-python.nix ({ pkgs, ... }: -let - # A suitable k3s pause image, also used for the test pod - pauseImage = pkgs.dockerTools.buildImage { - name = "test.local/pause"; - tag = "local"; - contents = with pkgs; [ tini coreutils busybox ]; - config.Entrypoint = [ "/bin/tini" "--" "/bin/sleep" "inf" ]; - }; - # Don't use the default service account because there's a race where it may - # not be created yet; make our own instead. - testPodYaml = pkgs.writeText "test.yml" '' - apiVersion: v1 - kind: ServiceAccount - metadata: - name: test - --- - apiVersion: v1 - kind: Pod - metadata: - name: test - spec: - serviceAccountName: test - containers: - - name: test - image: test.local/pause:local - imagePullPolicy: Never - command: ["sh", "-c", "sleep inf"] - ''; -in -{ - name = "k3s"; - meta = with pkgs.lib.maintainers; { - maintainers = [ euank ]; - }; - - machine = { pkgs, ... }: { - environment.systemPackages = with pkgs; [ k3s gzip ]; - - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = pkgs.lib.mkDefault 1536; - virtualisation.diskSize = pkgs.lib.mkDefault 4096; - - services.k3s = { - enable = true; - role = "server"; - docker = true; - # Slightly reduce resource usage - extraFlags = "--no-deploy coredns,servicelb,traefik,local-storage,metrics-server --pause-image test.local/pause:local"; + let + # A suitable k3s pause image, also used for the test pod + pauseImage = pkgs.dockerTools.buildImage { + name = "test.local/pause"; + tag = "local"; + contents = with pkgs; [ tini coreutils busybox ]; + config.Entrypoint = [ "/bin/tini" "--" "/bin/sleep" "inf" ]; + }; + # Don't use the default service account because there's a race where it may + # not be created yet; make our own instead. + testPodYaml = pkgs.writeText "test.yml" '' + apiVersion: v1 + kind: ServiceAccount + metadata: + name: test + --- + apiVersion: v1 + kind: Pod + metadata: + name: test + spec: + serviceAccountName: test + containers: + - name: test + image: test.local/pause:local + imagePullPolicy: Never + command: ["sh", "-c", "sleep inf"] + ''; + in + { + name = "k3s"; + meta = with pkgs.lib.maintainers; { + maintainers = [ euank ]; }; - users.users = { - noprivs = { - isNormalUser = true; - description = "Can't access k3s by default"; - password = "*"; + machine = { pkgs, ... }: { + environment.systemPackages = with pkgs; [ k3s gzip ]; + + # k3s uses enough resources the default vm fails. + virtualisation.memorySize = pkgs.lib.mkDefault 1536; + virtualisation.diskSize = pkgs.lib.mkDefault 4096; + + services.k3s = { + enable = true; + role = "server"; + docker = true; + # Slightly reduce resource usage + extraFlags = "--no-deploy coredns,servicelb,traefik,local-storage,metrics-server --pause-image test.local/pause:local"; + }; + + users.users = { + noprivs = { + isNormalUser = true; + description = "Can't access k3s by default"; + password = "*"; + }; }; }; - }; - testScript = '' - start_all() + testScript = '' + start_all() - machine.wait_for_unit("k3s") - machine.succeed("k3s kubectl cluster-info") - machine.fail("sudo -u noprivs k3s kubectl cluster-info") - # machine.succeed("k3s check-config") # fails with the current nixos kernel config, uncomment once this passes + machine.wait_for_unit("k3s") + machine.succeed("k3s kubectl cluster-info") + machine.fail("sudo -u noprivs k3s kubectl cluster-info") + # machine.succeed("k3s check-config") # fails with the current nixos kernel config, uncomment once this passes - machine.succeed( - "zcat ${pauseImage} | docker load" - ) + machine.succeed( + "zcat ${pauseImage} | docker load" + ) - machine.succeed("k3s kubectl apply -f ${testPodYaml}") - machine.succeed("k3s kubectl wait --for 'condition=Ready' pod/test") - machine.succeed("k3s kubectl delete -f ${testPodYaml}") + machine.succeed("k3s kubectl apply -f ${testPodYaml}") + machine.succeed("k3s kubectl wait --for 'condition=Ready' pod/test") + machine.succeed("k3s kubectl delete -f ${testPodYaml}") - machine.shutdown() - ''; -}) + machine.shutdown() + ''; + }) diff --git a/nixos/tests/k3s.nix b/nixos/tests/k3s.nix index ff329af3c5ec..dfebca50677a 100644 --- a/nixos/tests/k3s.nix +++ b/nixos/tests/k3s.nix @@ -1,78 +1,78 @@ import ./make-test-python.nix ({ pkgs, ... }: -let - # A suitable k3s pause image, also used for the test pod - pauseImage = pkgs.dockerTools.buildImage { - name = "test.local/pause"; - tag = "local"; - contents = with pkgs; [ tini coreutils busybox ]; - config.Entrypoint = [ "/bin/tini" "--" "/bin/sleep" "inf" ]; - }; - # Don't use the default service account because there's a race where it may - # not be created yet; make our own instead. - testPodYaml = pkgs.writeText "test.yml" '' - apiVersion: v1 - kind: ServiceAccount - metadata: - name: test - --- - apiVersion: v1 - kind: Pod - metadata: - name: test - spec: - serviceAccountName: test - containers: - - name: test - image: test.local/pause:local - imagePullPolicy: Never - command: ["sh", "-c", "sleep inf"] - ''; -in -{ - name = "k3s"; - meta = with pkgs.lib.maintainers; { - maintainers = [ euank ]; - }; + let + # A suitable k3s pause image, also used for the test pod + pauseImage = pkgs.dockerTools.buildImage { + name = "test.local/pause"; + tag = "local"; + contents = with pkgs; [ tini coreutils busybox ]; + config.Entrypoint = [ "/bin/tini" "--" "/bin/sleep" "inf" ]; + }; + # Don't use the default service account because there's a race where it may + # not be created yet; make our own instead. + testPodYaml = pkgs.writeText "test.yml" '' + apiVersion: v1 + kind: ServiceAccount + metadata: + name: test + --- + apiVersion: v1 + kind: Pod + metadata: + name: test + spec: + serviceAccountName: test + containers: + - name: test + image: test.local/pause:local + imagePullPolicy: Never + command: ["sh", "-c", "sleep inf"] + ''; + in + { + name = "k3s"; + meta = with pkgs.lib.maintainers; { + maintainers = [ euank ]; + }; - machine = { pkgs, ... }: { - environment.systemPackages = with pkgs; [ k3s gzip ]; + machine = { pkgs, ... }: { + environment.systemPackages = with pkgs; [ k3s gzip ]; - # k3s uses enough resources the default vm fails. - virtualisation.memorySize = pkgs.lib.mkDefault 1536; - virtualisation.diskSize = pkgs.lib.mkDefault 4096; + # k3s uses enough resources the default vm fails. + virtualisation.memorySize = pkgs.lib.mkDefault 1536; + virtualisation.diskSize = pkgs.lib.mkDefault 4096; - services.k3s.enable = true; - services.k3s.role = "server"; - services.k3s.package = pkgs.k3s; - # Slightly reduce resource usage - services.k3s.extraFlags = "--no-deploy coredns,servicelb,traefik,local-storage,metrics-server --pause-image test.local/pause:local"; + services.k3s.enable = true; + services.k3s.role = "server"; + services.k3s.package = pkgs.k3s; + # Slightly reduce resource usage + services.k3s.extraFlags = "--no-deploy coredns,servicelb,traefik,local-storage,metrics-server --pause-image test.local/pause:local"; - users.users = { - noprivs = { - isNormalUser = true; - description = "Can't access k3s by default"; - password = "*"; + users.users = { + noprivs = { + isNormalUser = true; + description = "Can't access k3s by default"; + password = "*"; + }; }; }; - }; - testScript = '' - start_all() + testScript = '' + start_all() - machine.wait_for_unit("k3s") - machine.succeed("k3s kubectl cluster-info") - machine.fail("sudo -u noprivs k3s kubectl cluster-info") - # machine.succeed("k3s check-config") # fails with the current nixos kernel config, uncomment once this passes + machine.wait_for_unit("k3s") + machine.succeed("k3s kubectl cluster-info") + machine.fail("sudo -u noprivs k3s kubectl cluster-info") + # machine.succeed("k3s check-config") # fails with the current nixos kernel config, uncomment once this passes - machine.succeed( - "zcat ${pauseImage} | k3s ctr image import -" - ) + machine.succeed( + "zcat ${pauseImage} | k3s ctr image import -" + ) - machine.succeed("k3s kubectl apply -f ${testPodYaml}") - machine.succeed("k3s kubectl wait --for 'condition=Ready' pod/test") - machine.succeed("k3s kubectl delete -f ${testPodYaml}") + machine.succeed("k3s kubectl apply -f ${testPodYaml}") + machine.succeed("k3s kubectl wait --for 'condition=Ready' pod/test") + machine.succeed("k3s kubectl delete -f ${testPodYaml}") - machine.shutdown() - ''; -}) + machine.shutdown() + ''; + }) From 3c61779d5c15de7e9a34e3f0eb8e59f4dbadc1cc Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Mon, 25 Oct 2021 17:43:27 -0700 Subject: [PATCH 5/5] nixos/tests/k3s: address pr feedback --- ...s-docker.nix => k3s-single-node-docker.nix} | 18 +++++++++++------- nixos/tests/{k3s.nix => k3s-single-node.nix} | 18 +++++++++++------- 2 files changed, 22 insertions(+), 14 deletions(-) rename nixos/tests/{k3s-docker.nix => k3s-single-node-docker.nix} (79%) rename nixos/tests/{k3s.nix => k3s-single-node.nix} (79%) diff --git a/nixos/tests/k3s-docker.nix b/nixos/tests/k3s-single-node-docker.nix similarity index 79% rename from nixos/tests/k3s-docker.nix rename to nixos/tests/k3s-single-node-docker.nix index 62544e3ee13e..7f3d15788b04 100644 --- a/nixos/tests/k3s-docker.nix +++ b/nixos/tests/k3s-single-node-docker.nix @@ -1,11 +1,14 @@ import ./make-test-python.nix ({ pkgs, ... }: let - # A suitable k3s pause image, also used for the test pod - pauseImage = pkgs.dockerTools.buildImage { + imageEnv = pkgs.buildEnv { + name = "k3s-pause-image-env"; + paths = with pkgs; [ tini (hiPrio coreutils) busybox ]; + }; + pauseImage = pkgs.dockerTools.streamLayeredImage { name = "test.local/pause"; tag = "local"; - contents = with pkgs; [ tini coreutils busybox ]; + contents = imageEnv; config.Entrypoint = [ "/bin/tini" "--" "/bin/sleep" "inf" ]; }; # Don't use the default service account because there's a race where it may @@ -39,8 +42,8 @@ import ./make-test-python.nix ({ pkgs, ... }: environment.systemPackages = with pkgs; [ k3s gzip ]; # k3s uses enough resources the default vm fails. - virtualisation.memorySize = pkgs.lib.mkDefault 1536; - virtualisation.diskSize = pkgs.lib.mkDefault 4096; + virtualisation.memorySize = 1536; + virtualisation.diskSize = 4096; services.k3s = { enable = true; @@ -65,10 +68,11 @@ import ./make-test-python.nix ({ pkgs, ... }: machine.wait_for_unit("k3s") machine.succeed("k3s kubectl cluster-info") machine.fail("sudo -u noprivs k3s kubectl cluster-info") - # machine.succeed("k3s check-config") # fails with the current nixos kernel config, uncomment once this passes + # FIXME: this fails with the current nixos kernel config; once it passes, we should uncomment it + # machine.succeed("k3s check-config") machine.succeed( - "zcat ${pauseImage} | docker load" + "${pauseImage} | docker load" ) machine.succeed("k3s kubectl apply -f ${testPodYaml}") diff --git a/nixos/tests/k3s.nix b/nixos/tests/k3s-single-node.nix similarity index 79% rename from nixos/tests/k3s.nix rename to nixos/tests/k3s-single-node.nix index dfebca50677a..d98f20d468cb 100644 --- a/nixos/tests/k3s.nix +++ b/nixos/tests/k3s-single-node.nix @@ -1,11 +1,14 @@ import ./make-test-python.nix ({ pkgs, ... }: let - # A suitable k3s pause image, also used for the test pod - pauseImage = pkgs.dockerTools.buildImage { + imageEnv = pkgs.buildEnv { + name = "k3s-pause-image-env"; + paths = with pkgs; [ tini (hiPrio coreutils) busybox ]; + }; + pauseImage = pkgs.dockerTools.streamLayeredImage { name = "test.local/pause"; tag = "local"; - contents = with pkgs; [ tini coreutils busybox ]; + contents = imageEnv; config.Entrypoint = [ "/bin/tini" "--" "/bin/sleep" "inf" ]; }; # Don't use the default service account because there's a race where it may @@ -39,8 +42,8 @@ import ./make-test-python.nix ({ pkgs, ... }: environment.systemPackages = with pkgs; [ k3s gzip ]; # k3s uses enough resources the default vm fails. - virtualisation.memorySize = pkgs.lib.mkDefault 1536; - virtualisation.diskSize = pkgs.lib.mkDefault 4096; + virtualisation.memorySize = 1536; + virtualisation.diskSize = 4096; services.k3s.enable = true; services.k3s.role = "server"; @@ -63,10 +66,11 @@ import ./make-test-python.nix ({ pkgs, ... }: machine.wait_for_unit("k3s") machine.succeed("k3s kubectl cluster-info") machine.fail("sudo -u noprivs k3s kubectl cluster-info") - # machine.succeed("k3s check-config") # fails with the current nixos kernel config, uncomment once this passes + # FIXME: this fails with the current nixos kernel config; once it passes, we should uncomment it + # machine.succeed("k3s check-config") machine.succeed( - "zcat ${pauseImage} | k3s ctr image import -" + "${pauseImage} | k3s ctr image import -" ) machine.succeed("k3s kubectl apply -f ${testPodYaml}")