nixos/kubernetes: adapt module and test cases to fit kubernetes v1.20.X as well as coredns v1.7.X

nixos/modules/services/cluster/kubernetes/addons/dns.nix

@@ -3,7 +3,7 @@
 with lib;
 
 let
-  version = "1.6.4";
+  version = "1.7.1";
   cfg = config.services.kubernetes.addons.dns;
   ports = {
     dns = 10053;
@@ -55,9 +55,9 @@ in {
       type = types.attrs;
       default = {
         imageName = "coredns/coredns";
-        imageDigest = "sha256:493ee88e1a92abebac67cbd4b5658b4730e0f33512461442d8d9214ea6734a9b";
+        imageDigest = "sha256:4a6e0769130686518325b21b0c1d0688b54e7c79244d48e1b15634e98e40c6ef";
         finalImageTag = version;
-        sha256 = "0fm9zdjavpf5hni8g7fkdd3csjbhd7n7py7llxjc66sbii087028";
+        sha256 = "02r440xcdsgi137k5lmmvp0z5w5fmk8g9mysq5pnysq1wl8sj6mw";
       };
     };
   };
@@ -156,7 +156,6 @@ in {
             health :${toString ports.health}
             kubernetes ${cfg.clusterDomain} in-addr.arpa ip6.arpa {
               pods insecure
-              upstream
               fallthrough in-addr.arpa ip6.arpa
             }
             prometheus :${toString ports.metrics}

nixos/modules/services/cluster/kubernetes/apiserver.nix

@@ -238,14 +238,42 @@ in
       type = int;
     };
 
+    apiAudiences = mkOption {
+      description = ''
+        Kubernetes apiserver ServiceAccount issuer.
+      '';
+      default = "api,https://kubernetes.default.svc";
+      type = str;
+    };
+
+    serviceAccountIssuer = mkOption {
+      description = ''
+        Kubernetes apiserver ServiceAccount issuer.
+      '';
+      default = "https://kubernetes.default.svc";
+      type = str;
+    };
+
+    serviceAccountSigningKeyFile = mkOption {
+      description = ''
+        Path to the file that contains the current private key of the service
+        account token issuer. The issuer will sign issued ID tokens with this
+        private key.
+      '';
+      default = top.serviceAccountSigningKeyFile;
+      type = path;
+    };
+
     serviceAccountKeyFile = mkOption {
       description = ''
-        Kubernetes apiserver PEM-encoded x509 RSA private or public key file,
+        File containing PEM-encoded x509 RSA or ECDSA private or public keys,
-        used to verify ServiceAccount tokens. By default tls private key file
+        used to verify ServiceAccount tokens. The specified file can contain
-        is used.
+        multiple keys, and the flag can be specified multiple times with
+        different files. If unspecified, --tls-private-key-file is used.
+        Must be specified when --service-account-signing-key is provided
       '';
-      default = null;
+      default = top.serviceAccountKeyFile;
-      type = nullOr path;
+      type = path;
     };
 
     serviceClusterIpRange = mkOption {
@@ -357,8 +385,10 @@ in
               ${optionalString (cfg.runtimeConfig != "")
                 "--runtime-config=${cfg.runtimeConfig}"} \
               --secure-port=${toString cfg.securePort} \
-              ${optionalString (cfg.serviceAccountKeyFile!=null)
+              --api-audiences=${toString cfg.apiAudiences} \
-                "--service-account-key-file=${cfg.serviceAccountKeyFile}"} \
+              --service-account-issuer=${toString cfg.serviceAccountIssuer} \
+              --service-account-signing-key-file=${cfg.serviceAccountSigningKeyFile} \
+              --service-account-key-file=${cfg.serviceAccountKeyFile} \
               --service-cluster-ip-range=${cfg.serviceClusterIpRange} \
               --storage-backend=${cfg.storageBackend} \
               ${optionalString (cfg.tlsCertFile != null)

nixos/modules/services/cluster/kubernetes/kubelet.nix

@@ -125,6 +125,18 @@ in
       };
     };
 
+    containerRuntime = mkOption {
+      description = "Which container runtime type to use";
+      type = enum ["docker" "remote"];
+      default = "remote";
+    };
+
+    containerRuntimeEndpoint = mkOption {
+      description = "Endpoint at which to find the container runtime api interface/socket";
+      type = str;
+      default = "unix:///var/run/docker/containerd/containerd.sock";
+    };
+
     enable = mkEnableOption "Kubernetes kubelet.";
 
     extraOpts = mkOption {
@@ -240,7 +252,7 @@ in
       systemd.services.kubelet = {
         description = "Kubernetes Kubelet Service";
         wantedBy = [ "kubernetes.target" ];
-        after = [ "network.target" "docker.service" "kube-apiserver.service" ];
+        after = [ "network.target" "kube-apiserver.service" "sockets.target" ];
         path = with pkgs; [
           gitMinimal
           openssh
@@ -306,6 +318,8 @@ in
             ${optionalString (cfg.tlsKeyFile != null)
               "--tls-private-key-file=${cfg.tlsKeyFile}"} \
             ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
+            --container-runtime=${cfg.containerRuntime} \
+            --container-runtime-endpoint=${cfg.containerRuntimeEndpoint} \
             ${cfg.extraOpts}
           '';
           WorkingDirectory = top.dataDir;

nixos/modules/services/cluster/kubernetes/pki.nix

@@ -361,6 +361,7 @@ in
           tlsCertFile = mkDefault cert;
           tlsKeyFile = mkDefault key;
           serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.cert;
+          serviceAccountSigningKeyFile = mkDefault cfg.certs.serviceAccount.key;
           kubeletClientCaFile = mkDefault caCert;
           kubeletClientCertFile = mkDefault cfg.certs.apiserverKubeletClient.cert;
           kubeletClientKeyFile = mkDefault cfg.certs.apiserverKubeletClient.key;

pkgs/applications/networking/cluster/kubernetes/default.nix

@@ -33,6 +33,8 @@ stdenv.mkDerivation rec {
 
   outputs = [ "out" "man" "pause" ];
 
+  patches = [ ./fixup-addonmanager-lib-path.patch ];
+
   postPatch = ''
     # go env breaks the sandbox
     substituteInPlace "hack/lib/golang.sh" \
@@ -64,10 +66,17 @@ stdenv.mkDerivation rec {
     install -D build/pause/linux/pause -t $pause/bin
     installManPage docs/man/man1/*.[1-9]
 
-    cp cluster/addons/addon-manager/kube-addons.sh $out/bin/kube-addons
+    # Unfortunately, kube-addons-main.sh only looks for the lib file in either the current working dir
+    # or in /opt. We have to patch this for now.
+    substitute cluster/addons/addon-manager/kube-addons-main.sh $out/bin/kube-addons \
+      --subst-var out
+
+    chmod +x $out/bin/kube-addons
     patchShebangs $out/bin/kube-addons
     wrapProgram $out/bin/kube-addons --set "KUBECTL_BIN" "$out/bin/kubectl"
 
+    cp cluster/addons/addon-manager/kube-addons.sh $out/bin/kube-addons-lib.sh
+
     cp ${./mk-docker-opts.sh} $out/bin/mk-docker-opts.sh
 
     for tool in kubeadm kubectl; do

pkgs/applications/networking/cluster/kubernetes/fixup-addonmanager-lib-path.patch

@@ -0,0 +1,23 @@
+diff --git a/cluster/addons/addon-manager/kube-addons-main.sh b/cluster/addons/addon-manager/kube-addons-main.sh
+index 849973470d1..e4fef30eaea 100755
+--- a/cluster/addons/addon-manager/kube-addons-main.sh
++++ b/cluster/addons/addon-manager/kube-addons-main.sh
+@@ -17,17 +17,7 @@
+ # Import required functions. The addon manager is installed to /opt in
+ # production use (see the Dockerfile)
+ # Disabling shellcheck following files as the full path would be required.
+-if [ -f "kube-addons.sh" ]; then
+-  # shellcheck disable=SC1091
+-  source "kube-addons.sh"
+-elif [ -f "/opt/kube-addons.sh" ]; then
+-  # shellcheck disable=SC1091
+-  source "/opt/kube-addons.sh"
+-else
+-  # If the required source is missing, we have to fail.
+-  log ERR "== Could not find kube-addons.sh (not in working directory or /opt) at $(date -Is) =="
+-  exit 1
+-fi
++source "@out@/bin/kube-addons-lib.sh"
+ 
+ # The business logic for whether a given object should be created
+ # was already enforced by salt, and /etc/kubernetes/addons is the