nixpkgs/nixos/tests/kubernetes/base.nix

{ system ? builtins.currentSystem,
  config ? {},
  pkgs ? import ../../.. { inherit system config; }
}:

with import ../../lib/testing-python.nix { inherit system pkgs; };
with pkgs.lib;

let
  mkKubernetesBaseTest =
    { name, domain ? "my.zyx", test, machines
    , extraConfiguration ? null }:
    let
      masterName = head (filter (machineName: any (role: role == "master") machines.${machineName}.roles) (attrNames machines));
      master = machines.${masterName};
      extraHosts = ''
        ${master.ip}  etcd.${domain}
        ${master.ip}  api.${domain}
        ${concatMapStringsSep "\n" (machineName: "${machines.${machineName}.ip}  ${machineName}.${domain}") (attrNames machines)}
      '';
      kubectl = with pkgs; runCommand "wrap-kubectl" { buildInputs = [ makeWrapper ]; } ''
        mkdir -p $out/bin
        makeWrapper ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl --set KUBECONFIG "/etc/kubernetes/cluster-admin.kubeconfig"
      '';
    in makeTest {
      inherit name;

      nodes = mapAttrs (machineName: machine:
        { config, pkgs, lib, nodes, ... }:
          mkMerge [
            {
              boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
              virtualisation.memorySize = mkDefault 1536;
              virtualisation.diskSize = mkDefault 4096;
              networking = {
                inherit domain extraHosts;
                primaryIPAddress = mkForce machine.ip;

                firewall = {
                  allowedTCPPorts = [
                    10250 # kubelet
                  ];
                  trustedInterfaces = ["mynet"];

                  extraCommands = concatMapStrings  (node: ''
                    iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT
                  '') (attrValues nodes);
                };
              };
              programs.bash.enableCompletion = true;
              environment.systemPackages = [ kubectl ];
              services.flannel.iface = "eth1";
              services.kubernetes = {
                addons.dashboard.enable = true;
                proxy.hostname = "${masterName}.${domain}";

                easyCerts = true;
                inherit (machine) roles;
                apiserver = {
                  securePort = 443;
                  advertiseAddress = master.ip;
                };
                masterAddress = "${masterName}.${config.networking.domain}";
                # workaround for:
                #   https://github.com/kubernetes/kubernetes/issues/102676
                #   (workaround from) https://github.com/kubernetes/kubernetes/issues/95488
                kubelet.extraOpts = ''\
                  --cgroups-per-qos=false \
                  --enforce-node-allocatable="" \
                '';
              };
            }
            (optionalAttrs (any (role: role == "master") machine.roles) {
              networking.firewall.allowedTCPPorts = [
                443 # kubernetes apiserver
              ];
            })
            (optionalAttrs (machine ? extraConfiguration) (machine.extraConfiguration { inherit config pkgs lib nodes; }))
            (optionalAttrs (extraConfiguration != null) (extraConfiguration { inherit config pkgs lib nodes; }))
          ]
      ) machines;

      testScript = ''
        start_all()
      '' + test;
    };

  mkKubernetesMultiNodeTest = attrs: mkKubernetesBaseTest ({
    machines = {
      machine1 = {
        roles = ["master"];
        ip = "192.168.1.1";
      };
      machine2 = {
        roles = ["node"];
        ip = "192.168.1.2";
      };
    };
  } // attrs // {
    name = "kubernetes-${attrs.name}-multinode";
  });

  mkKubernetesSingleNodeTest = attrs: mkKubernetesBaseTest ({
    machines = {
      machine1 = {
        roles = ["master" "node"];
        ip = "192.168.1.1";
      };
    };
  } // attrs // {
    name = "kubernetes-${attrs.name}-singlenode";
  });
in {
  inherit mkKubernetesBaseTest mkKubernetesSingleNodeTest mkKubernetesMultiNodeTest;
}