From f9e2f76a590d11cbeaa10e3953ddc96110bf1b3b Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Fri, 1 Mar 2019 07:56:59 +0100
Subject: [PATCH 01/24] nixos/kubernetes: Add systemd path units

to protect services from crashing and clobbering the logs when
certificates are not in place yet and make sure services are activated
when certificates are ready.

To prevent errors similar to "kube-controller-manager.path: Failed to
enter waiting state: Too many open files"
fs.inotify.max_user_instances has to be increased.
---
 .../services/cluster/kubernetes/apiserver.nix | 41 ++++++++-
 .../cluster/kubernetes/controller-manager.nix | 22 ++++-
 .../services/cluster/kubernetes/flannel.nix   |  8 +-
 .../services/cluster/kubernetes/kubelet.nix   | 19 ++++-
 .../services/cluster/kubernetes/pki.nix       | 84 ++++++++++++++++++-
 nixos/tests/kubernetes/base.nix               |  5 +-
 6 files changed, 168 insertions(+), 11 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix
index 81e45b417de3..08f929060aa0 100644
--- a/nixos/modules/services/cluster/kubernetes/apiserver.nix
+++ b/nixos/modules/services/cluster/kubernetes/apiserver.nix
@@ -272,7 +272,25 @@ in
   ###### implementation
   config = mkMerge [
 
-    (mkIf cfg.enable {
+    (mkIf cfg.enable (let
+      apiserverPaths = [
+        cfg.clientCaFile
+        cfg.etcd.caFile
+        cfg.etcd.certFile
+        cfg.etcd.keyFile
+        cfg.kubeletClientCaFile
+        cfg.kubeletClientCertFile
+        cfg.kubeletClientKeyFile
+        cfg.serviceAccountKeyFile
+        cfg.tlsCertFile
+        cfg.tlsKeyFile
+      ];
+      etcdPaths = [
+        config.services.etcd.certFile
+        config.services.etcd.keyFile
+        config.services.etcd.trustedCaFile
+      ];
+    in {
         systemd.services.kube-apiserver = {
           description = "Kubernetes APIServer Service";
           wantedBy = [ "kubernetes.target" ];
@@ -341,6 +359,25 @@ in
             Restart = "on-failure";
             RestartSec = 5;
           };
+          unitConfig.ConditionPathExists = apiserverPaths;
+        };
+
+        systemd.paths.kube-apiserver = {
+          wantedBy = [ "kube-apiserver.service" ];
+          pathConfig = {
+            PathExists = apiserverPaths;
+            PathChanged = apiserverPaths;
+          };
+        };
+
+        systemd.services.etcd.unitConfig.ConditionPathExists = etcdPaths;
+
+        systemd.paths.etcd = {
+          wantedBy = [ "etcd.service" ];
+          pathConfig = {
+            PathExists = etcdPaths;
+            PathChanged = etcdPaths;
+          };
         };
 
         services.etcd = {
@@ -421,7 +458,7 @@ in
         };
       };
 
-    })
+    }))
 
   ];
 
diff --git a/nixos/modules/services/cluster/kubernetes/controller-manager.nix b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
index dff97f144d55..27b28311adbc 100644
--- a/nixos/modules/services/cluster/kubernetes/controller-manager.nix
+++ b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
@@ -104,7 +104,16 @@ in
   };
 
   ###### implementation
-  config = mkIf cfg.enable {
+  config = mkIf cfg.enable (let
+    controllerManagerPaths = [
+      cfg.rootCaFile
+      cfg.tlsCertFile
+      cfg.tlsKeyFile
+      top.pki.certs.controllerManagerClient.cert
+      top.pki.certs.controllerManagerClient.key
+    ];
+  in {
+
     systemd.services.kube-controller-manager = {
       description = "Kubernetes Controller Manager Service";
       wantedBy = [ "kubernetes.target" ];
@@ -142,6 +151,15 @@ in
         Group = "kubernetes";
       };
       path = top.path;
+      unitConfig.ConditionPathExists = controllerManagerPaths;
+    };
+
+    systemd.paths.kube-controller-manager = {
+      wantedBy = [ "kube-controller-manager.service" ];
+      pathConfig = {
+        PathExists = controllerManagerPaths;
+        PathChanged = controllerManagerPaths;
+      };
     };
 
     services.kubernetes.pki.certs = with top.lib; {
@@ -158,5 +176,5 @@ in
     };
 
     services.kubernetes.controllerManager.kubeconfig.server = mkDefault top.apiserverAddress;
-  };
+  });
 }
diff --git a/nixos/modules/services/cluster/kubernetes/flannel.nix b/nixos/modules/services/cluster/kubernetes/flannel.nix
index 93ee2fd65eeb..ef06acb6de39 100644
--- a/nixos/modules/services/cluster/kubernetes/flannel.nix
+++ b/nixos/modules/services/cluster/kubernetes/flannel.nix
@@ -55,13 +55,15 @@ in
         ${mkDockerOpts}/mk-docker-opts -d /run/flannel/docker
         systemctl restart docker
       '';
+      unitConfig.ConditionPathExists = [ "/run/flannel/subnet.env" ];
       serviceConfig.Type = "oneshot";
     };
 
-    systemd.paths."flannel-subnet-env" = {
-      wantedBy = [ "flannel.service" ];
+    systemd.paths.flannel-subnet-env = {
+      wantedBy = [ "mk-docker-opts.service" ];
       pathConfig = {
-        PathModified = "/run/flannel/subnet.env";
+        PathExists = [ "/run/flannel/subnet.env" ];
+        PathChanged = [ "/run/flannel/subnet.env" ];
         Unit = "mk-docker-opts.service";
       };
     };
diff --git a/nixos/modules/services/cluster/kubernetes/kubelet.nix b/nixos/modules/services/cluster/kubernetes/kubelet.nix
index c94bb28bf7fb..86402cba7c48 100644
--- a/nixos/modules/services/cluster/kubernetes/kubelet.nix
+++ b/nixos/modules/services/cluster/kubernetes/kubelet.nix
@@ -241,7 +241,13 @@ in
 
   ###### implementation
   config = mkMerge [
-    (mkIf cfg.enable {
+    (mkIf cfg.enable (let
+      kubeletPaths = [
+        cfg.clientCaFile
+        cfg.tlsCertFile
+        cfg.tlsKeyFile
+      ];
+    in {
       services.kubernetes.kubelet.seedDockerImages = [infraContainer];
 
       systemd.services.kubelet = {
@@ -308,6 +314,15 @@ in
           '';
           WorkingDirectory = top.dataDir;
         };
+        unitConfig.ConditionPathExists = kubeletPaths;
+      };
+
+      systemd.paths.kubelet = {
+        wantedBy =  [ "kubelet.service" ];
+        pathConfig = {
+          PathExists = kubeletPaths;
+          PathChanged = kubeletPaths;
+        };
       };
 
       # Allways include cni plugins
@@ -336,7 +351,7 @@ in
       };
 
       services.kubernetes.kubelet.kubeconfig.server = mkDefault top.apiserverAddress;
-    })
+    }))
 
     (mkIf (cfg.enable && cfg.manifests != {}) {
       environment.etc = mapAttrs' (name: manifest:
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 38deca23a990..8ad17d4dfb4e 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -119,6 +119,29 @@ in
     cfsslCertPathPrefix = "${config.services.cfssl.dataDir}/cfssl";
     cfsslCert = "${cfsslCertPathPrefix}.pem";
     cfsslKey = "${cfsslCertPathPrefix}-key.pem";
+
+    certmgrPaths = [
+      top.caFile
+      certmgrAPITokenPath
+    ];
+    addonManagerPaths = mkIf top.addonManager.enable [
+      cfg.certs.addonManager.cert
+      cfg.certs.addonManager.key
+      cfg.certs.clusterAdmin.cert
+      cfg.certs.clusterAdmin.key
+    ];
+    flannelPaths = [
+      cfg.certs.flannelClient.cert
+      cfg.certs.flannelClient.key
+    ];
+    proxyPaths = mkIf top.proxy.enable [
+      cfg.certs.kubeProxyClient.cert
+      cfg.certs.kubeProxyClient.key
+    ];
+    schedulerPaths = mkIf top.scheduler.enable [
+      cfg.certs.schedulerClient.cert
+      cfg.certs.schedulerClient.key
+    ];
   in
   {
 
@@ -230,6 +253,18 @@ in
           mapAttrs mkSpec cfg.certs;
       };
 
+      systemd.services.certmgr = {
+        unitConfig.ConditionPathExists = certmgrPaths;
+      };
+
+      systemd.paths.certmgr = {
+        wantedBy = [ "certmgr.service" ];
+        pathConfig = {
+          PathExists = certmgrPaths;
+          PathChanged = certmgrPaths;
+        };
+      };
+
       #TODO: Get rid of kube-addon-manager in the future for the following reasons
       # - it is basically just a shell script wrapped around kubectl
       # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount
@@ -255,7 +290,18 @@ in
             export KUBECONFIG=${clusterAdminKubeconfig}
             ${kubectl}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files}
           '';
-        })]);
+        })
+        {
+          unitConfig.ConditionPathExists = addonManagerPaths;
+        }]);
+
+      systemd.paths.kube-addon-manager = mkIf top.addonManager.enable {
+        wantedBy = [ "kube-addon-manager.service" ];
+        pathConfig = {
+          PathExists = addonManagerPaths;
+          PathChanged = addonManagerPaths;
+        };
+      };
 
       environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (!isNull cfg.etcClusterAdminKubeconfig)
         clusterAdminKubeconfig;
@@ -337,6 +383,42 @@ in
         };
       };
 
+      systemd.services.flannel = {
+        unitConfig.ConditionPathExists = flannelPaths;
+      };
+
+      systemd.paths.flannel = {
+        wantedBy = [ "flannel.service" ];
+        pathConfig = {
+          PathExists = flannelPaths;
+          PathChanged = flannelPaths;
+        };
+      };
+
+      systemd.services.kube-proxy = mkIf top.proxy.enable {
+        unitConfig.ConditionPathExists = proxyPaths;
+      };
+
+      systemd.paths.kube-proxy = mkIf top.proxy.enable {
+        wantedBy = [ "kube-proxy.service" ];
+        pathConfig = {
+          PathExists = proxyPaths;
+          PathChanged = proxyPaths;
+        };
+      };
+
+      systemd.services.kube-scheduler = mkIf top.scheduler.enable {
+        unitConfig.ConditionPathExists = schedulerPaths;
+      };
+
+      systemd.paths.kube-scheduler = mkIf top.scheduler.enable {
+        wantedBy = [ "kube-scheduler.service" ];
+        pathConfig = {
+          PathExists = schedulerPaths;
+          PathChanged = schedulerPaths;
+        };
+      };
+
       services.kubernetes = {
 
         apiserver = mkIf top.apiserver.enable (with cfg.certs.apiServer; {
diff --git a/nixos/tests/kubernetes/base.nix b/nixos/tests/kubernetes/base.nix
index ec1a75e74c41..212023859f6d 100644
--- a/nixos/tests/kubernetes/base.nix
+++ b/nixos/tests/kubernetes/base.nix
@@ -30,7 +30,10 @@ let
         { config, pkgs, lib, nodes, ... }:
           mkMerge [
             {
-              boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
+              boot = {
+                postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
+                kernel.sysctl = { "fs.inotify.max_user_instances" = 256; };
+              };
               virtualisation.memorySize = mkDefault 1536;
               virtualisation.diskSize = mkDefault 4096;
               networking = {

From 62f03750e48ae7658ea18d7ac75833279da02a5a Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Fri, 1 Mar 2019 08:44:45 +0100
Subject: [PATCH 02/24] nixos/kubernetes: Stabilize services startup across
 machines
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

by adding targets and curl wait loops to services to ensure services
are not started before their depended services are reachable.

Extra targets cfssl-online.target and kube-apiserver-online.target
syncronize starts across machines and node-online.target ensures
docker is restarted and ready to deploy containers on after flannel
has discussed the network cidr with apiserver.

Since flannel needs to be started before addon-manager to configure
the docker interface, it has to have its own rbac bootstrap service.

The curl wait loops within the other services exists to ensure that when
starting the service it is able to do its work immediately without
clobbering the log about failing conditions.

By ensuring kubernetes.target is only reached after starting the
cluster it can be used in the tests as a wait condition.

In kube-certmgr-bootstrap mkdir is needed for it to not fail to start.

The following is the relevant part of systemctl list-dependencies

default.target
● ├─certmgr.service
● ├─cfssl.service
● ├─docker.service
● ├─etcd.service
● ├─flannel.service
● ├─kubernetes.target
● │ ├─kube-addon-manager.service
● │ ├─kube-proxy.service
● │ ├─kube-apiserver-online.target
● │ │ ├─flannel-rbac-bootstrap.service
● │ │ ├─kube-apiserver-online.service
● │ │ ├─kube-apiserver.service
● │ │ ├─kube-controller-manager.service
● │ │ └─kube-scheduler.service
● │ └─node-online.target
● │   ├─node-online.service
● │   ├─flannel.target
● │   │ ├─flannel.service
● │   │ └─mk-docker-opts.service
● │   └─kubelet.target
● │     └─kubelet.service
● ├─network-online.target
● │ └─cfssl-online.target
● │   ├─certmgr.service
● │   ├─cfssl-online.service
● │   └─kube-certmgr-bootstrap.service
---
 .../cluster/kubernetes/addon-manager.nix      |   2 +-
 .../services/cluster/kubernetes/apiserver.nix |  26 +++-
 .../cluster/kubernetes/controller-manager.nix |  11 +-
 .../services/cluster/kubernetes/default.nix   |  13 ++
 .../services/cluster/kubernetes/flannel.nix   | 117 ++++++++++++------
 .../services/cluster/kubernetes/kubelet.nix   |  40 +++++-
 .../services/cluster/kubernetes/pki.nix       |  65 +++++++++-
 .../services/cluster/kubernetes/proxy.nix     |  10 +-
 .../services/cluster/kubernetes/scheduler.nix |  11 +-
 nixos/tests/kubernetes/dns.nix                |   3 +
 nixos/tests/kubernetes/rbac.nix               |   4 +
 11 files changed, 251 insertions(+), 51 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/addon-manager.nix b/nixos/modules/services/cluster/kubernetes/addon-manager.nix
index 17f2dde31a71..abd9e99ba02a 100644
--- a/nixos/modules/services/cluster/kubernetes/addon-manager.nix
+++ b/nixos/modules/services/cluster/kubernetes/addon-manager.nix
@@ -72,7 +72,7 @@ in
     systemd.services.kube-addon-manager = {
       description = "Kubernetes addon manager";
       wantedBy = [ "kubernetes.target" ];
-      after = [ "kube-apiserver.service" ];
+      after = [ "kube-apiserver-online.target" "node-online.target" ];
       environment.ADDON_PATH = "/etc/kubernetes/addons/";
       path = [ pkgs.gawk ];
       serviceConfig = {
diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix
index 08f929060aa0..567d31f06ef7 100644
--- a/nixos/modules/services/cluster/kubernetes/apiserver.nix
+++ b/nixos/modules/services/cluster/kubernetes/apiserver.nix
@@ -293,8 +293,9 @@ in
     in {
         systemd.services.kube-apiserver = {
           description = "Kubernetes APIServer Service";
-          wantedBy = [ "kubernetes.target" ];
-          after = [ "network.target" ];
+          wantedBy = [ "kube-apiserver-online.target" ];
+          after = [ "certmgr.service" ];
+          before = [ "kube-apiserver-online.target" ];
           serviceConfig = {
             Slice = "kubernetes.slice";
             ExecStart = ''${top.package}/bin/kube-apiserver \
@@ -459,7 +460,28 @@ in
       };
 
     }))
+    {
+      systemd.targets.kube-apiserver-online = {
+        wantedBy = [ "kubernetes.target" ];
+        before = [ "kubernetes.target" ];
+      };
 
+      systemd.services.kube-apiserver-online = mkIf top.flannel.enable {
+        description = "apiserver control plane is online";
+        wantedBy = [ "kube-apiserver-online.target" ];
+        after = [ "kube-scheduler.service" "kube-controller-manager.service" ];
+        before = [ "kube-apiserver-online.target" ];
+        preStart = ''
+          ${top.lib.mkWaitCurl (with top.pki.certs.flannelClient; {
+            sleep = 3;
+            path = "/healthz";
+            cacert = top.caFile;
+            inherit cert key;
+          })}
+        '';
+        script = "echo apiserver control plane is online";
+      };
+    }
   ];
 
 }
diff --git a/nixos/modules/services/cluster/kubernetes/controller-manager.nix b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
index 27b28311adbc..20f471215dba 100644
--- a/nixos/modules/services/cluster/kubernetes/controller-manager.nix
+++ b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
@@ -116,8 +116,17 @@ in
 
     systemd.services.kube-controller-manager = {
       description = "Kubernetes Controller Manager Service";
-      wantedBy = [ "kubernetes.target" ];
+      wantedBy = [ "kube-apiserver-online.target" ];
       after = [ "kube-apiserver.service" ];
+      before = [ "kube-apiserver-online.target" ];
+      preStart = ''
+        ${top.lib.mkWaitCurl (with top.pki.certs.controllerManagerClient; {
+          sleep = 1;
+          path = "/api";
+          cacert = top.caFile;
+          inherit cert key;
+        })}
+      '';
       serviceConfig = {
         RestartSec = "30s";
         Restart = "on-failure";
diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix
index 375e33e91b5a..f1f544afc4d1 100644
--- a/nixos/modules/services/cluster/kubernetes/default.nix
+++ b/nixos/modules/services/cluster/kubernetes/default.nix
@@ -73,6 +73,18 @@ let
     };
   };
 
+  mkWaitCurl = { address ? cfg.apiserverAddress, sleep ? 2, path ? "", args ? "-o /dev/null",
+                 cacert ? null, cert ? null, key ? null, }: ''
+    while ! ${pkgs.curl}/bin/curl --fail-early -fs \
+      ${if cacert != null then "--cacert ${cacert}" else ""} \
+      ${if cert != null then "--cert ${cert}" else ""} \
+      ${if key != null then "--key ${key}" else ""} \
+      ${address}${path} ${args} ; do
+        sleep ${toString sleep}
+        echo Waiting to be able to reach ${address}${path}
+    done
+  '';
+
   kubeConfigDefaults = {
     server = mkDefault cfg.kubeconfig.server;
     caFile = mkDefault cfg.kubeconfig.caFile;
@@ -162,6 +174,7 @@ in {
         inherit mkCert;
         inherit mkKubeConfig;
         inherit mkKubeConfigOptions;
+        inherit mkWaitCurl;
       };
       type = types.attrs;
     };
diff --git a/nixos/modules/services/cluster/kubernetes/flannel.nix b/nixos/modules/services/cluster/kubernetes/flannel.nix
index ef06acb6de39..4aa547c9d3e4 100644
--- a/nixos/modules/services/cluster/kubernetes/flannel.nix
+++ b/nixos/modules/services/cluster/kubernetes/flannel.nix
@@ -27,7 +27,12 @@ in
   };
 
   ###### implementation
-  config = mkIf cfg.enable {
+  config = mkIf cfg.enable (let
+    flannelBootstrapPaths = mkIf top.apiserver.enable [
+      top.pki.certs.clusterAdmin.cert
+      top.pki.certs.clusterAdmin.key
+    ];
+  in {
     services.flannel = {
 
       enable = mkDefault true;
@@ -48,8 +53,10 @@ in
       }];
     };
 
-    systemd.services."mk-docker-opts" = {
+    systemd.services.mk-docker-opts = {
       description = "Pre-Docker Actions";
+      wantedBy = [ "flannel.target" ];
+      before = [ "flannel.target" ];
       path = with pkgs; [ gawk gnugrep ];
       script = ''
         ${mkDockerOpts}/mk-docker-opts -d /run/flannel/docker
@@ -68,6 +75,17 @@ in
       };
     };
 
+    systemd.targets.flannel = {
+      wantedBy = [ "node-online.target" ];
+      before = [ "node-online.target" ];
+    };
+
+    systemd.services.flannel = {
+      wantedBy = [ "flannel.target" ];
+      after = [ "kubelet.target" ];
+      before = [ "flannel.target" ];
+    };
+
     systemd.services.docker = {
       environment.DOCKER_OPTS = "-b none";
       serviceConfig.EnvironmentFile = "-/run/flannel/docker";
@@ -93,44 +111,69 @@ in
     };
 
     # give flannel som kubernetes rbac permissions if applicable
-    services.kubernetes.addonManager.bootstrapAddons = mkIf ((storageBackend == "kubernetes") && (elem "RBAC" top.apiserver.authorizationMode)) {
+    systemd.services.flannel-rbac-bootstrap = mkIf (top.apiserver.enable && (elem "RBAC" top.apiserver.authorizationMode)) {
 
-      flannel-cr = {
-        apiVersion = "rbac.authorization.k8s.io/v1beta1";
-        kind = "ClusterRole";
-        metadata = { name = "flannel"; };
-        rules = [{
-          apiGroups = [ "" ];
-          resources = [ "pods" ];
-          verbs = [ "get" ];
-        }
-        {
-          apiGroups = [ "" ];
-          resources = [ "nodes" ];
-          verbs = [ "list" "watch" ];
-        }
-        {
-          apiGroups = [ "" ];
-          resources = [ "nodes/status" ];
-          verbs = [ "patch" ];
-        }];
-      };
+      wantedBy = [ "kube-apiserver-online.target" ];
+      after = [ "kube-apiserver-online.target" ];
+      before = [ "flannel.service" ];
+      path = with pkgs; [ kubectl ];
+      preStart = let
+        files = mapAttrsToList (n: v: pkgs.writeText "${n}.json" (builtins.toJSON v)) {
+          flannel-cr = {
+            apiVersion = "rbac.authorization.k8s.io/v1beta1";
+            kind = "ClusterRole";
+            metadata = { name = "flannel"; };
+            rules = [{
+              apiGroups = [ "" ];
+              resources = [ "pods" ];
+              verbs = [ "get" ];
+            }
+            {
+              apiGroups = [ "" ];
+              resources = [ "nodes" ];
+              verbs = [ "list" "watch" ];
+            }
+            {
+              apiGroups = [ "" ];
+              resources = [ "nodes/status" ];
+              verbs = [ "patch" ];
+            }];
+          };
 
-      flannel-crb = {
-        apiVersion = "rbac.authorization.k8s.io/v1beta1";
-        kind = "ClusterRoleBinding";
-        metadata = { name = "flannel"; };
-        roleRef = {
-          apiGroup = "rbac.authorization.k8s.io";
-          kind = "ClusterRole";
-          name = "flannel";
+          flannel-crb = {
+            apiVersion = "rbac.authorization.k8s.io/v1beta1";
+            kind = "ClusterRoleBinding";
+            metadata = { name = "flannel"; };
+            roleRef = {
+              apiGroup = "rbac.authorization.k8s.io";
+              kind = "ClusterRole";
+              name = "flannel";
+            };
+            subjects = [{
+              kind = "User";
+              name = "flannel-client";
+            }];
+          };
         };
-        subjects = [{
-          kind = "User";
-          name = "flannel-client";
-        }];
-      };
+      in ''
+        ${top.lib.mkWaitCurl (with top.pki.certs.clusterAdmin; {
+          path = "/";
+          cacert = top.caFile;
+          inherit cert key;
+        })}
 
+        kubectl -s ${top.apiserverAddress} --certificate-authority=${top.caFile} --client-certificate=${top.pki.certs.clusterAdmin.cert} --client-key=${top.pki.certs.clusterAdmin.key} apply -f ${concatStringsSep " \\\n -f " files}
+      '';
+      script = "echo Ok";
+      unitConfig.ConditionPathExists = flannelBootstrapPaths;
     };
-  };
+
+    systemd.paths.flannel-rbac-bootstrap = mkIf top.apiserver.enable {
+      wantedBy = [ "flannel-rbac-bootstrap.service" ];
+      pathConfig = {
+        PathExists = flannelBootstrapPaths;
+        PathChanged = flannelBootstrapPaths;
+      };
+    };
+  });
 }
diff --git a/nixos/modules/services/cluster/kubernetes/kubelet.nix b/nixos/modules/services/cluster/kubernetes/kubelet.nix
index 86402cba7c48..b3f3c0365642 100644
--- a/nixos/modules/services/cluster/kubernetes/kubelet.nix
+++ b/nixos/modules/services/cluster/kubernetes/kubelet.nix
@@ -252,8 +252,9 @@ in
 
       systemd.services.kubelet = {
         description = "Kubernetes Kubelet Service";
-        wantedBy = [ "kubernetes.target" ];
-        after = [ "network.target" "docker.service" "kube-apiserver.service" ];
+        wantedBy = [ "kubelet.target" ];
+        after = [ "kube-apiserver-online.target" ];
+        before = [ "kubelet.target" ];
         path = with pkgs; [ gitMinimal openssh docker utillinux iproute ethtool thin-provisioning-tools iptables socat ] ++ top.path;
         preStart = ''
           ${concatMapStrings (img: ''
@@ -325,6 +326,30 @@ in
         };
       };
 
+      systemd.services.docker.before = [ "kubelet.service" ];
+
+      systemd.services.node-online = {
+        wantedBy = [ "node-online.target" ];
+        after = [ "flannel.target" "kubelet.target" ];
+        before = [ "node-online.target" ];
+        # it is complicated. flannel needs kubelet to run the pause container before
+        # it discusses the node CIDR with apiserver and afterwards configures and restarts
+        # dockerd. Until then prevent creating any pods because they have to be recreated anyway
+        # because the network of docker0 has been changed by flannel.
+        script = let
+          docker-env = "/run/flannel/docker";
+          flannel-date = "stat --print=%Y ${docker-env}";
+          docker-date = "systemctl show --property=ActiveEnterTimestamp --value docker";
+        in ''
+          while ! test -f ${docker-env} ; do sleep 1 ; done
+          while test `${flannel-date}` -gt `date +%s --date="$(${docker-date})"` ; do
+            sleep 1
+          done
+        '';
+        serviceConfig.Type = "oneshot";
+        serviceConfig.Slice = "kubernetes.slice";
+      };
+
       # Allways include cni plugins
       services.kubernetes.kubelet.cni.packages = [pkgs.cni-plugins];
 
@@ -369,5 +394,16 @@ in
       };
     })
 
+    {
+      systemd.targets.kubelet = {
+        wantedBy = [ "node-online.target" ];
+        before = [ "node-online.target" ];
+      };
+
+      systemd.targets.node-online = {
+        wantedBy = [ "kubernetes.target" ];
+        before = [ "kubernetes.target" ];
+      };
+    }
   ];
 }
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 8ad17d4dfb4e..d08d7892bb53 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -119,6 +119,7 @@ in
     cfsslCertPathPrefix = "${config.services.cfssl.dataDir}/cfssl";
     cfsslCert = "${cfsslCertPathPrefix}.pem";
     cfsslKey = "${cfsslCertPathPrefix}-key.pem";
+    cfsslPort = toString config.services.cfssl.port;
 
     certmgrPaths = [
       top.caFile
@@ -191,13 +192,39 @@ in
         chown cfssl "${cfsslAPITokenPath}" && chmod 400 "${cfsslAPITokenPath}"
       '')]);
 
+    systemd.targets.cfssl-online = {
+      wantedBy = [ "network-online.target" ];
+      after = [ "cfssl.service" "network-online.target" "cfssl-online.service" ];
+    };
+
+    systemd.services.cfssl-online = {
+      description = "Wait for ${remote} to be reachable.";
+      wantedBy = [ "cfssl-online.target" ];
+      before = [ "cfssl-online.target" ];
+      preStart = ''
+        ${top.lib.mkWaitCurl {
+          address = remote;
+          path = "/api/v1/cfssl/info";
+          args = "-kd '{}' -o /dev/null";
+        }}
+      '';
+      script = "echo Ok";
+      serviceConfig = {
+        TimeoutSec = "300";
+      };
+    };
+
     systemd.services.kube-certmgr-bootstrap = {
       description = "Kubernetes certmgr bootstrapper";
-      wantedBy = [ "certmgr.service" ];
-      after = [ "cfssl.target" ];
+      wantedBy = [ "cfssl-online.target" ];
+      after = [ "cfssl-online.target" ];
+      before = [ "certmgr.service" ];
       script = concatStringsSep "\n" [''
         set -e
 
+        mkdir -p $(dirname ${certmgrAPITokenPath})
+        mkdir -p $(dirname ${top.caFile})
+
         # If there's a cfssl (cert issuer) running locally, then don't rely on user to
         # manually paste it in place. Just symlink.
         # otherwise, create the target file, ready for users to insert the token
@@ -209,14 +236,18 @@ in
         fi
       ''
       (optionalString (cfg.pkiTrustOnBootstrap) ''
-        if [ ! -f "${top.caFile}" ] || [ $(cat "${top.caFile}" | wc -c) -lt 1 ]; then
-          ${pkgs.curl}/bin/curl --fail-early -f -kd '{}' ${remote}/api/v1/cfssl/info | \
-            ${pkgs.cfssl}/bin/cfssljson -stdout >${top.caFile}
+        if [ ! -s "${top.caFile}" ]; then
+          ${top.lib.mkWaitCurl {
+            address = "https://${top.masterAddress}:${cfsslPort}";
+            path = "/api/v1/cfssl/info";
+            args = "-kd '{}' -o - | ${pkgs.cfssl}/bin/cfssljson -stdout >${top.caFile}";
+          }}
         fi
       '')
       ];
       serviceConfig = {
-        RestartSec = "10s";
+        TimeoutSec = "300";
+        RestartSec = "1s";
         Restart = "on-failure";
       };
     };
@@ -254,6 +285,14 @@ in
       };
 
       systemd.services.certmgr = {
+        wantedBy = [ "cfssl-online.target" ];
+        after = [ "cfssl-online.target" "kube-certmgr-bootstrap.service" ];
+        preStart = ''
+          while ! test -s ${certmgrAPITokenPath} ; do
+            sleep 1
+            echo Waiting for ${certmgrAPITokenPath}
+          done
+        '';
         unitConfig.ConditionPathExists = certmgrPaths;
       };
 
@@ -289,6 +328,12 @@ in
           ''
             export KUBECONFIG=${clusterAdminKubeconfig}
             ${kubectl}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files}
+
+            ${top.lib.mkWaitCurl (with top.pki.certs.addonManager; {
+              path = "/api/v1/namespaces/kube-system/serviceaccounts/default";
+              cacert = top.caFile;
+              inherit cert key;
+            })}
           '';
         })
         {
@@ -384,6 +429,14 @@ in
       };
 
       systemd.services.flannel = {
+        preStart = ''
+          ${top.lib.mkWaitCurl (with top.pki.certs.flannelClient; {
+            path = "/api/v1/nodes";
+            cacert = top.caFile;
+            inherit cert key;
+            args = "-o - | grep podCIDR >/dev/null";
+          })}
+        '';
         unitConfig.ConditionPathExists = flannelPaths;
       };
 
diff --git a/nixos/modules/services/cluster/kubernetes/proxy.nix b/nixos/modules/services/cluster/kubernetes/proxy.nix
index 83cd3e231000..073756d58abf 100644
--- a/nixos/modules/services/cluster/kubernetes/proxy.nix
+++ b/nixos/modules/services/cluster/kubernetes/proxy.nix
@@ -49,8 +49,16 @@ in
     systemd.services.kube-proxy = {
       description = "Kubernetes Proxy Service";
       wantedBy = [ "kubernetes.target" ];
-      after = [ "kube-apiserver.service" ];
+      after = [ "node-online.target" ];
+      before = [ "kubernetes.target" ];
       path = with pkgs; [ iptables conntrack_tools ];
+      preStart = ''
+        ${top.lib.mkWaitCurl (with top.pki.certs.kubeProxyClient; {
+          path = "/api/v1/nodes/${top.kubelet.hostname}";
+          cacert = top.caFile;
+          inherit cert key;
+        })}
+      '';
       serviceConfig = {
         Slice = "kubernetes.slice";
         ExecStart = ''${top.package}/bin/kube-proxy \
diff --git a/nixos/modules/services/cluster/kubernetes/scheduler.nix b/nixos/modules/services/cluster/kubernetes/scheduler.nix
index 0305b9aefe59..d3302a15402b 100644
--- a/nixos/modules/services/cluster/kubernetes/scheduler.nix
+++ b/nixos/modules/services/cluster/kubernetes/scheduler.nix
@@ -59,8 +59,17 @@ in
   config = mkIf cfg.enable {
     systemd.services.kube-scheduler = {
       description = "Kubernetes Scheduler Service";
-      wantedBy = [ "kubernetes.target" ];
+      wantedBy = [ "kube-apiserver-online.target" ];
       after = [ "kube-apiserver.service" ];
+      before = [ "kube-apiserver-online.target" ];
+      preStart = ''
+        ${top.lib.mkWaitCurl (with top.pki.certs.schedulerClient; {
+          sleep = 1;
+          path = "/api";
+          cacert = top.caFile;
+          inherit cert key;
+        })}
+      '';
       serviceConfig = {
         Slice = "kubernetes.slice";
         ExecStart = ''${top.package}/bin/kube-scheduler \
diff --git a/nixos/tests/kubernetes/dns.nix b/nixos/tests/kubernetes/dns.nix
index 46bcb01a5265..e7db0a58ab61 100644
--- a/nixos/tests/kubernetes/dns.nix
+++ b/nixos/tests/kubernetes/dns.nix
@@ -77,6 +77,7 @@ let
   singleNodeTest = {
     test = ''
       # prepare machine1 for test
+      $machine1->waitForUnit("kubernetes.target");
       $machine1->waitUntilSucceeds("kubectl get node machine1.${domain} | grep -w Ready");
       $machine1->waitUntilSucceeds("docker load < ${redisImage}");
       $machine1->waitUntilSucceeds("kubectl create -f ${redisPod}");
@@ -102,6 +103,8 @@ let
       # Node token exchange
       $machine1->waitUntilSucceeds("cp -f /var/lib/cfssl/apitoken.secret /tmp/shared/apitoken.secret");
       $machine2->waitUntilSucceeds("cat /tmp/shared/apitoken.secret | nixos-kubernetes-node-join");
+      $machine1->waitForUnit("kubernetes.target");
+      $machine2->waitForUnit("kubernetes.target");
 
       # prepare machines for test
       $machine1->waitUntilSucceeds("kubectl get node machine2.${domain} | grep -w Ready");
diff --git a/nixos/tests/kubernetes/rbac.nix b/nixos/tests/kubernetes/rbac.nix
index 3ce7adcd0d71..967fe506004f 100644
--- a/nixos/tests/kubernetes/rbac.nix
+++ b/nixos/tests/kubernetes/rbac.nix
@@ -94,6 +94,8 @@ let
 
   singlenode = base // {
     test = ''
+      $machine1->waitForUnit("kubernetes.target");
+
       $machine1->waitUntilSucceeds("kubectl get node machine1.my.zyx | grep -w Ready");
 
       $machine1->waitUntilSucceeds("docker load < ${kubectlImage}");
@@ -116,6 +118,8 @@ let
       # Node token exchange
       $machine1->waitUntilSucceeds("cp -f /var/lib/cfssl/apitoken.secret /tmp/shared/apitoken.secret");
       $machine2->waitUntilSucceeds("cat /tmp/shared/apitoken.secret | nixos-kubernetes-node-join");
+      $machine1->waitForUnit("kubernetes.target");
+      $machine2->waitForUnit("kubernetes.target");
 
       $machine1->waitUntilSucceeds("kubectl get node machine2.my.zyx | grep -w Ready");
 

From 51aeaaffc2a108268f8f9b633f3291115f15e72d Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Sun, 3 Mar 2019 19:06:46 +0100
Subject: [PATCH 03/24] nixos/kubernetes: flannel needs iptables in service
 path

---
 nixos/modules/services/cluster/kubernetes/flannel.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/nixos/modules/services/cluster/kubernetes/flannel.nix b/nixos/modules/services/cluster/kubernetes/flannel.nix
index 4aa547c9d3e4..a5b4f7103dcb 100644
--- a/nixos/modules/services/cluster/kubernetes/flannel.nix
+++ b/nixos/modules/services/cluster/kubernetes/flannel.nix
@@ -84,6 +84,7 @@ in
       wantedBy = [ "flannel.target" ];
       after = [ "kubelet.target" ];
       before = [ "flannel.target" ];
+      path = [ pkgs.iptables ];
     };
 
     systemd.services.docker = {

From cf8389c9048b9e46dadc47142f85a6086098e064 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Sun, 3 Mar 2019 19:07:22 +0100
Subject: [PATCH 04/24] nixos/kubernetes: Add longer timeouts for waiting
 services

---
 nixos/modules/services/cluster/kubernetes/apiserver.nix | 3 +++
 nixos/modules/services/cluster/kubernetes/pki.nix       | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix
index 567d31f06ef7..2e7e2a6ec3e8 100644
--- a/nixos/modules/services/cluster/kubernetes/apiserver.nix
+++ b/nixos/modules/services/cluster/kubernetes/apiserver.nix
@@ -480,6 +480,9 @@ in
           })}
         '';
         script = "echo apiserver control plane is online";
+        serviceConfig = {
+          TimeoutSec = "500";
+        };
       };
     }
   ];
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index d08d7892bb53..6f3f41072074 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -246,7 +246,7 @@ in
       '')
       ];
       serviceConfig = {
-        TimeoutSec = "300";
+        TimeoutSec = "500";
         RestartSec = "1s";
         Restart = "on-failure";
       };

From fd28c0a82a56da127ad8e77d34941b57e9e46065 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Fri, 1 Mar 2019 10:23:34 +0100
Subject: [PATCH 05/24] nixos/kubernetes: Seed docker images before kubelet
 service start

to speed up startup time because it can be parallelized.
---
 .../services/cluster/kubernetes/kubelet.nix   | 23 ++++++++++++++-----
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/kubelet.nix b/nixos/modules/services/cluster/kubernetes/kubelet.nix
index b3f3c0365642..0b1d2a67565c 100644
--- a/nixos/modules/services/cluster/kubernetes/kubelet.nix
+++ b/nixos/modules/services/cluster/kubernetes/kubelet.nix
@@ -257,12 +257,7 @@ in
         before = [ "kubelet.target" ];
         path = with pkgs; [ gitMinimal openssh docker utillinux iproute ethtool thin-provisioning-tools iptables socat ] ++ top.path;
         preStart = ''
-          ${concatMapStrings (img: ''
-            echo "Seeding docker image: ${img}"
-            docker load <${img}
-          '') cfg.seedDockerImages}
-
-          rm /opt/cni/bin/* || true
+          rm -f /opt/cni/bin/* || true
           ${concatMapStrings (package: ''
             echo "Linking cni package: ${package}"
             ln -fs ${package}/bin/* /opt/cni/bin
@@ -328,6 +323,22 @@ in
 
       systemd.services.docker.before = [ "kubelet.service" ];
 
+      systemd.services.docker-seed-images = {
+        wantedBy = [ "docker.service" ];
+        after = [ "docker.service" ];
+        before = [ "kubelet.service" ];
+        path = with pkgs; [ docker ];
+        preStart = ''
+          ${concatMapStrings (img: ''
+            echo "Seeding docker image: ${img}"
+            docker load <${img}
+          '') cfg.seedDockerImages}
+        '';
+        script = "echo Ok";
+        serviceConfig.Type = "oneshot";
+        serviceConfig.Slice = "kubernetes.slice";
+      };
+
       systemd.services.node-online = {
         wantedBy = [ "node-online.target" ];
         after = [ "flannel.target" "kubelet.target" ];

From 7df88bd802c939cb1118bd9c3208999796dc0795 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Fri, 1 Mar 2019 10:28:29 +0100
Subject: [PATCH 06/24] nixos/kubernetes: Put dashboard service account into
 bootstrapAddons

to prevent errors in log about missing permissions when
addon manager starts the dashboard.
---
 .../cluster/kubernetes/addons/dashboard.nix   | 36 +++++++++++--------
 1 file changed, 21 insertions(+), 15 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/addons/dashboard.nix b/nixos/modules/services/cluster/kubernetes/addons/dashboard.nix
index 454e7d35bc01..2295694ffc74 100644
--- a/nixos/modules/services/cluster/kubernetes/addons/dashboard.nix
+++ b/nixos/modules/services/cluster/kubernetes/addons/dashboard.nix
@@ -169,6 +169,23 @@ in {
         };
       };
 
+      kubernetes-dashboard-cm = {
+        apiVersion = "v1";
+        kind = "ConfigMap";
+        metadata = {
+          labels = {
+            k8s-app = "kubernetes-dashboard";
+            # Allows editing resource and makes sure it is created first.
+            "addonmanager.kubernetes.io/mode" = "EnsureExists";
+          };
+          name = "kubernetes-dashboard-settings";
+          namespace = "kube-system";
+        };
+      };
+    };
+
+    services.kubernetes.addonManager.bootstrapAddons = mkMerge [{
+
       kubernetes-dashboard-sa = {
         apiVersion = "v1";
         kind = "ServiceAccount";
@@ -210,20 +227,9 @@ in {
         };
         type = "Opaque";
       };
-      kubernetes-dashboard-cm = {
-        apiVersion = "v1";
-        kind = "ConfigMap";
-        metadata = {
-          labels = {
-            k8s-app = "kubernetes-dashboard";
-            # Allows editing resource and makes sure it is created first.
-            "addonmanager.kubernetes.io/mode" = "EnsureExists";
-          };
-          name = "kubernetes-dashboard-settings";
-          namespace = "kube-system";
-        };
-      };
-    } // (optionalAttrs cfg.rbac.enable
+    }
+
+    (optionalAttrs cfg.rbac.enable
       (let
         subjects = [{
           kind = "ServiceAccount";
@@ -323,6 +329,6 @@ in {
             inherit subjects;
           };
         })
-    ));
+    ))];
   };
 }

From 74962bf767b67ca8b92e82fb0d6f6f96927601d0 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Fri, 1 Mar 2019 10:43:26 +0100
Subject: [PATCH 07/24] nixos/kubernetes: No need to restart services besides
 certmgr

within the node join script, since certmgr is taking care of
restarting services.
---
 .../services/cluster/kubernetes/pki.nix       | 30 +++++--------------
 1 file changed, 7 insertions(+), 23 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 6f3f41072074..329278e375ca 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -375,33 +375,17 @@ in
           exit 1
         fi
 
+        do_restart=$(test -s ${certmgrAPITokenPath} && echo -n y || echo -n n)
+
         echo $token > ${certmgrAPITokenPath}
         chmod 600 ${certmgrAPITokenPath}
 
-        echo "Restarting certmgr..." >&1
-        systemctl restart certmgr
+        if [ y = $do_restart ]; then
+          echo "Restarting certmgr..." >&1
+          systemctl restart certmgr
+        fi
 
-        echo "Waiting for certs to appear..." >&1
-
-        ${optionalString top.kubelet.enable ''
-          while [ ! -f ${cfg.certs.kubelet.cert} ]; do sleep 1; done
-          echo "Restarting kubelet..." >&1
-          systemctl restart kubelet
-        ''}
-
-        ${optionalString top.proxy.enable ''
-          while [ ! -f ${cfg.certs.kubeProxyClient.cert} ]; do sleep 1; done
-          echo "Restarting kube-proxy..." >&1
-          systemctl restart kube-proxy
-        ''}
-
-        ${optionalString top.flannel.enable ''
-          while [ ! -f ${cfg.certs.flannelClient.cert} ]; do sleep 1; done
-          echo "Restarting flannel..." >&1
-          systemctl restart flannel
-        ''}
-
-        echo "Node joined succesfully"
+        echo "Node joined succesfully" >&1
       '')];
 
       # isolate etcd on loopback at the master node

From ff91d5818cf4703e01670251096da301cc2c7c54 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Wed, 6 Mar 2019 16:40:27 +0100
Subject: [PATCH 08/24] nixos/kubernetes: Address review: Rename targets and
 move proxy to node-online.target

---
 .../cluster/kubernetes/addon-manager.nix         |  4 ++--
 .../services/cluster/kubernetes/apiserver.nix    | 16 ++++++++--------
 .../cluster/kubernetes/controller-manager.nix    |  4 ++--
 .../services/cluster/kubernetes/kubelet.nix      |  4 ++--
 .../services/cluster/kubernetes/proxy.nix        |  6 +++---
 .../services/cluster/kubernetes/scheduler.nix    |  4 ++--
 6 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/addon-manager.nix b/nixos/modules/services/cluster/kubernetes/addon-manager.nix
index abd9e99ba02a..46f5b68b2a5c 100644
--- a/nixos/modules/services/cluster/kubernetes/addon-manager.nix
+++ b/nixos/modules/services/cluster/kubernetes/addon-manager.nix
@@ -71,8 +71,8 @@ in
 
     systemd.services.kube-addon-manager = {
       description = "Kubernetes addon manager";
-      wantedBy = [ "kubernetes.target" ];
-      after = [ "kube-apiserver-online.target" "node-online.target" ];
+      wantedBy = [ "kube-control-plane-online.target" ];
+      before = [ "kube-control-plane-online.target" ];
       environment.ADDON_PATH = "/etc/kubernetes/addons/";
       path = [ pkgs.gawk ];
       serviceConfig = {
diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix
index 2e7e2a6ec3e8..e4650c12cff9 100644
--- a/nixos/modules/services/cluster/kubernetes/apiserver.nix
+++ b/nixos/modules/services/cluster/kubernetes/apiserver.nix
@@ -293,9 +293,9 @@ in
     in {
         systemd.services.kube-apiserver = {
           description = "Kubernetes APIServer Service";
-          wantedBy = [ "kube-apiserver-online.target" ];
+          wantedBy = [ "kube-control-plane-online.target" ];
           after = [ "certmgr.service" ];
-          before = [ "kube-apiserver-online.target" ];
+          before = [ "kube-control-plane-online.target" ];
           serviceConfig = {
             Slice = "kubernetes.slice";
             ExecStart = ''${top.package}/bin/kube-apiserver \
@@ -461,16 +461,16 @@ in
 
     }))
     {
-      systemd.targets.kube-apiserver-online = {
+      systemd.targets.kube-control-plane-online = {
         wantedBy = [ "kubernetes.target" ];
         before = [ "kubernetes.target" ];
       };
 
-      systemd.services.kube-apiserver-online = mkIf top.flannel.enable {
-        description = "apiserver control plane is online";
-        wantedBy = [ "kube-apiserver-online.target" ];
+      systemd.services.kube-control-plane-online = rec {
+        description = "Kubernetes control plane is online";
+        wantedBy = [ "kube-control-plane-online.target" ];
         after = [ "kube-scheduler.service" "kube-controller-manager.service" ];
-        before = [ "kube-apiserver-online.target" ];
+        before = [ "kube-control-plane-online.target" ];
         preStart = ''
           ${top.lib.mkWaitCurl (with top.pki.certs.flannelClient; {
             sleep = 3;
@@ -479,7 +479,7 @@ in
             inherit cert key;
           })}
         '';
-        script = "echo apiserver control plane is online";
+        script = "echo Ok";
         serviceConfig = {
           TimeoutSec = "500";
         };
diff --git a/nixos/modules/services/cluster/kubernetes/controller-manager.nix b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
index 20f471215dba..8e82db36425a 100644
--- a/nixos/modules/services/cluster/kubernetes/controller-manager.nix
+++ b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
@@ -116,9 +116,9 @@ in
 
     systemd.services.kube-controller-manager = {
       description = "Kubernetes Controller Manager Service";
-      wantedBy = [ "kube-apiserver-online.target" ];
+      wantedBy = [ "kube-control-plane-online.target" ];
       after = [ "kube-apiserver.service" ];
-      before = [ "kube-apiserver-online.target" ];
+      before = [ "kube-control-plane-online.target" ];
       preStart = ''
         ${top.lib.mkWaitCurl (with top.pki.certs.controllerManagerClient; {
           sleep = 1;
diff --git a/nixos/modules/services/cluster/kubernetes/kubelet.nix b/nixos/modules/services/cluster/kubernetes/kubelet.nix
index 0b1d2a67565c..c4ecb2417e29 100644
--- a/nixos/modules/services/cluster/kubernetes/kubelet.nix
+++ b/nixos/modules/services/cluster/kubernetes/kubelet.nix
@@ -253,7 +253,7 @@ in
       systemd.services.kubelet = {
         description = "Kubernetes Kubelet Service";
         wantedBy = [ "kubelet.target" ];
-        after = [ "kube-apiserver-online.target" ];
+        after = [ "kube-control-plane-online.target" ];
         before = [ "kubelet.target" ];
         path = with pkgs; [ gitMinimal openssh docker utillinux iproute ethtool thin-provisioning-tools iptables socat ] ++ top.path;
         preStart = ''
@@ -339,7 +339,7 @@ in
         serviceConfig.Slice = "kubernetes.slice";
       };
 
-      systemd.services.node-online = {
+      systemd.services.kubelet-online = {
         wantedBy = [ "node-online.target" ];
         after = [ "flannel.target" "kubelet.target" ];
         before = [ "node-online.target" ];
diff --git a/nixos/modules/services/cluster/kubernetes/proxy.nix b/nixos/modules/services/cluster/kubernetes/proxy.nix
index 073756d58abf..d13d23e997b7 100644
--- a/nixos/modules/services/cluster/kubernetes/proxy.nix
+++ b/nixos/modules/services/cluster/kubernetes/proxy.nix
@@ -48,9 +48,9 @@ in
   config = mkIf cfg.enable {
     systemd.services.kube-proxy = {
       description = "Kubernetes Proxy Service";
-      wantedBy = [ "kubernetes.target" ];
-      after = [ "node-online.target" ];
-      before = [ "kubernetes.target" ];
+      wantedBy = [ "node-online.target" ];
+      after = [ "kubelet-online.service" ];
+      before = [ "node-online.target" ];
       path = with pkgs; [ iptables conntrack_tools ];
       preStart = ''
         ${top.lib.mkWaitCurl (with top.pki.certs.kubeProxyClient; {
diff --git a/nixos/modules/services/cluster/kubernetes/scheduler.nix b/nixos/modules/services/cluster/kubernetes/scheduler.nix
index d3302a15402b..4aea9e9b6bd0 100644
--- a/nixos/modules/services/cluster/kubernetes/scheduler.nix
+++ b/nixos/modules/services/cluster/kubernetes/scheduler.nix
@@ -59,9 +59,9 @@ in
   config = mkIf cfg.enable {
     systemd.services.kube-scheduler = {
       description = "Kubernetes Scheduler Service";
-      wantedBy = [ "kube-apiserver-online.target" ];
+      wantedBy = [ "kube-control-plane-online.target" ];
       after = [ "kube-apiserver.service" ];
-      before = [ "kube-apiserver-online.target" ];
+      before = [ "kube-control-plane-online.target" ];
       preStart = ''
         ${top.lib.mkWaitCurl (with top.pki.certs.schedulerClient; {
           sleep = 1;

From 6e9037fed0ce0b55ef37188ec1a58e18e196a780 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Wed, 6 Mar 2019 16:44:38 +0100
Subject: [PATCH 09/24] nixos/kubernetes: Address review: Move bootstrapping
 addons into own service

---
 .../cluster/kubernetes/addon-manager.nix      |  26 +++++
 .../services/cluster/kubernetes/flannel.nix   | 102 ++++++------------
 .../services/cluster/kubernetes/pki.nix       |  46 +++-----
 3 files changed, 78 insertions(+), 96 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/addon-manager.nix b/nixos/modules/services/cluster/kubernetes/addon-manager.nix
index 46f5b68b2a5c..406b20b0d8d8 100644
--- a/nixos/modules/services/cluster/kubernetes/addon-manager.nix
+++ b/nixos/modules/services/cluster/kubernetes/addon-manager.nix
@@ -72,9 +72,16 @@ in
     systemd.services.kube-addon-manager = {
       description = "Kubernetes addon manager";
       wantedBy = [ "kube-control-plane-online.target" ];
+      after = [ "kube-addon-manager-bootstrap.service" ];
       before = [ "kube-control-plane-online.target" ];
       environment.ADDON_PATH = "/etc/kubernetes/addons/";
       path = [ pkgs.gawk ];
+      preStart = ''
+        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-addon-manager; {
+          path = "/api/v1/namespaces/kube-system/serviceaccounts/default";
+          cacert = top.caFile;
+        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+      '';
       serviceConfig = {
         Slice = "kubernetes.slice";
         ExecStart = "${top.package}/bin/kube-addons";
@@ -86,6 +93,25 @@ in
       };
     };
 
+    systemd.services.kube-addon-manager-bootstrap = mkIf (top.apiserver.enable && top.addonManager.bootstrapAddons != {}) {
+      wantedBy = [ "kube-control-plane-online.target" ];
+      after = [ "kube-apiserver.service" ];
+      before = [ "kube-control-plane-online.target" ];
+      path = [ pkgs.kubectl ];
+      preStart = with pkgs; let
+        files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v))
+          cfg.bootstrapAddons;
+      in ''
+        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-addon-manager-bootstrap; {
+          path = "/api";
+          cacert = top.caFile;
+        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+
+        kubectl apply -f ${concatStringsSep " \\\n -f " files}
+      '';
+      script = "echo Ok";
+    };
+
     services.kubernetes.addonManager.bootstrapAddons = mkIf isRBACEnabled
     (let
       name = system:kube-addon-manager;
diff --git a/nixos/modules/services/cluster/kubernetes/flannel.nix b/nixos/modules/services/cluster/kubernetes/flannel.nix
index a5b4f7103dcb..fba70e3b9200 100644
--- a/nixos/modules/services/cluster/kubernetes/flannel.nix
+++ b/nixos/modules/services/cluster/kubernetes/flannel.nix
@@ -27,12 +27,7 @@ in
   };
 
   ###### implementation
-  config = mkIf cfg.enable (let
-    flannelBootstrapPaths = mkIf top.apiserver.enable [
-      top.pki.certs.clusterAdmin.cert
-      top.pki.certs.clusterAdmin.key
-    ];
-  in {
+  config = mkIf cfg.enable {
     services.flannel = {
 
       enable = mkDefault true;
@@ -112,69 +107,42 @@ in
     };
 
     # give flannel som kubernetes rbac permissions if applicable
-    systemd.services.flannel-rbac-bootstrap = mkIf (top.apiserver.enable && (elem "RBAC" top.apiserver.authorizationMode)) {
+    services.kubernetes.addonManager.bootstrapAddons = mkIf ((storageBackend == "kubernetes") && (elem "RBAC" top.apiserver.authorizationMode)) {
+      flannel-cr = {
+        apiVersion = "rbac.authorization.k8s.io/v1beta1";
+        kind = "ClusterRole";
+        metadata = { name = "flannel"; };
+        rules = [{
+          apiGroups = [ "" ];
+          resources = [ "pods" ];
+          verbs = [ "get" ];
+        }
+        {
+          apiGroups = [ "" ];
+          resources = [ "nodes" ];
+          verbs = [ "list" "watch" ];
+        }
+        {
+          apiGroups = [ "" ];
+          resources = [ "nodes/status" ];
+          verbs = [ "patch" ];
+        }];
+      };
 
-      wantedBy = [ "kube-apiserver-online.target" ];
-      after = [ "kube-apiserver-online.target" ];
-      before = [ "flannel.service" ];
-      path = with pkgs; [ kubectl ];
-      preStart = let
-        files = mapAttrsToList (n: v: pkgs.writeText "${n}.json" (builtins.toJSON v)) {
-          flannel-cr = {
-            apiVersion = "rbac.authorization.k8s.io/v1beta1";
-            kind = "ClusterRole";
-            metadata = { name = "flannel"; };
-            rules = [{
-              apiGroups = [ "" ];
-              resources = [ "pods" ];
-              verbs = [ "get" ];
-            }
-            {
-              apiGroups = [ "" ];
-              resources = [ "nodes" ];
-              verbs = [ "list" "watch" ];
-            }
-            {
-              apiGroups = [ "" ];
-              resources = [ "nodes/status" ];
-              verbs = [ "patch" ];
-            }];
-          };
-
-          flannel-crb = {
-            apiVersion = "rbac.authorization.k8s.io/v1beta1";
-            kind = "ClusterRoleBinding";
-            metadata = { name = "flannel"; };
-            roleRef = {
-              apiGroup = "rbac.authorization.k8s.io";
-              kind = "ClusterRole";
-              name = "flannel";
-            };
-            subjects = [{
-              kind = "User";
-              name = "flannel-client";
-            }];
-          };
+      flannel-crb = {
+        apiVersion = "rbac.authorization.k8s.io/v1beta1";
+        kind = "ClusterRoleBinding";
+        metadata = { name = "flannel"; };
+        roleRef = {
+          apiGroup = "rbac.authorization.k8s.io";
+          kind = "ClusterRole";
+          name = "flannel";
         };
-      in ''
-        ${top.lib.mkWaitCurl (with top.pki.certs.clusterAdmin; {
-          path = "/";
-          cacert = top.caFile;
-          inherit cert key;
-        })}
-
-        kubectl -s ${top.apiserverAddress} --certificate-authority=${top.caFile} --client-certificate=${top.pki.certs.clusterAdmin.cert} --client-key=${top.pki.certs.clusterAdmin.key} apply -f ${concatStringsSep " \\\n -f " files}
-      '';
-      script = "echo Ok";
-      unitConfig.ConditionPathExists = flannelBootstrapPaths;
-    };
-
-    systemd.paths.flannel-rbac-bootstrap = mkIf top.apiserver.enable {
-      wantedBy = [ "flannel-rbac-bootstrap.service" ];
-      pathConfig = {
-        PathExists = flannelBootstrapPaths;
-        PathChanged = flannelBootstrapPaths;
+        subjects = [{
+          kind = "User";
+          name = "flannel-client";
+        }];
       };
     };
-  });
+  };
 }
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 329278e375ca..4d97d8322cd4 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -304,41 +304,29 @@ in
         };
       };
 
+      systemd.services.kube-addon-manager-bootstrap = mkIf (top.apiserver.enable && top.addonManager.bootstrapAddons != {}) {
+        environment = {
+          KUBECONFIG = clusterAdminKubeconfig;
+          inherit (cfg.certs.clusterAdmin) cert key;
+        };
+      };
+
       #TODO: Get rid of kube-addon-manager in the future for the following reasons
       # - it is basically just a shell script wrapped around kubectl
       # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount
       # - it is designed to be used with k8s system components only
       # - it would be better with a more Nix-oriented way of managing addons
-      systemd.services.kube-addon-manager = mkIf top.addonManager.enable (mkMerge [{
-        environment.KUBECONFIG = with cfg.certs.addonManager;
-          top.lib.mkKubeConfig "addon-manager" {
-            server = top.apiserverAddress;
-            certFile = cert;
-            keyFile = key;
+      systemd.services.kube-addon-manager = mkIf top.addonManager.enable {
+        environment = with cfg.certs.addonManager; {
+          KUBECONFIG = top.lib.mkKubeConfig "kube-addon-manager" {
+              server = top.apiserverAddress;
+              certFile = cert;
+              keyFile = key;
           };
-        }
-
-        (optionalAttrs (top.addonManager.bootstrapAddons != {}) {
-          serviceConfig.PermissionsStartOnly = true;
-          preStart = with pkgs;
-          let
-            files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v))
-              top.addonManager.bootstrapAddons;
-          in
-          ''
-            export KUBECONFIG=${clusterAdminKubeconfig}
-            ${kubectl}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files}
-
-            ${top.lib.mkWaitCurl (with top.pki.certs.addonManager; {
-              path = "/api/v1/namespaces/kube-system/serviceaccounts/default";
-              cacert = top.caFile;
-              inherit cert key;
-            })}
-          '';
-        })
-        {
-          unitConfig.ConditionPathExists = addonManagerPaths;
-        }]);
+          inherit cert key;
+        };
+        unitConfig.ConditionPathExists = addonManagerPaths;
+      };
 
       systemd.paths.kube-addon-manager = mkIf top.addonManager.enable {
         wantedBy = [ "kube-addon-manager.service" ];

From 52fe1d2e7a9c154fe962f7b47ce008bf06cfe746 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Wed, 6 Mar 2019 16:50:35 +0100
Subject: [PATCH 10/24] nixos/kubernetes: Address review: Move controller
 manager paths into pki

---
 .../cluster/kubernetes/controller-manager.nix | 22 ++-----------------
 .../services/cluster/kubernetes/pki.nix       | 19 ++++++++++++++++
 2 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/controller-manager.nix b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
index 8e82db36425a..cc43a243df1c 100644
--- a/nixos/modules/services/cluster/kubernetes/controller-manager.nix
+++ b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
@@ -104,16 +104,7 @@ in
   };
 
   ###### implementation
-  config = mkIf cfg.enable (let
-    controllerManagerPaths = [
-      cfg.rootCaFile
-      cfg.tlsCertFile
-      cfg.tlsKeyFile
-      top.pki.certs.controllerManagerClient.cert
-      top.pki.certs.controllerManagerClient.key
-    ];
-  in {
-
+  config = mkIf cfg.enable {
     systemd.services.kube-controller-manager = {
       description = "Kubernetes Controller Manager Service";
       wantedBy = [ "kube-control-plane-online.target" ];
@@ -160,15 +151,6 @@ in
         Group = "kubernetes";
       };
       path = top.path;
-      unitConfig.ConditionPathExists = controllerManagerPaths;
-    };
-
-    systemd.paths.kube-controller-manager = {
-      wantedBy = [ "kube-controller-manager.service" ];
-      pathConfig = {
-        PathExists = controllerManagerPaths;
-        PathChanged = controllerManagerPaths;
-      };
     };
 
     services.kubernetes.pki.certs = with top.lib; {
@@ -185,5 +167,5 @@ in
     };
 
     services.kubernetes.controllerManager.kubeconfig.server = mkDefault top.apiserverAddress;
-  });
+  };
 }
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 4d97d8322cd4..98284fba12ac 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -143,6 +143,13 @@ in
       cfg.certs.schedulerClient.cert
       cfg.certs.schedulerClient.key
     ];
+    controllerManagerPaths = [
+      top.controllerManager.rootCaFile
+      top.controllerManager.tlsCertFile
+      top.controllerManager.tlsKeyFile
+      cfg.certs.controllerManagerClient.cert
+      cfg.certs.controllerManagerClient.key
+    ];
   in
   {
 
@@ -336,6 +343,18 @@ in
         };
       };
 
+      systemd.services.kube-controller-manager = mkIf top.controllerManager.enable {
+        unitConfig.ConditionPathExists = controllerManagerPaths;
+      };
+
+      systemd.paths.kube-controller-manager = mkIf top.controllerManager.enable {
+        wantedBy = [ "kube-controller-manager.service" ];
+        pathConfig = {
+          PathExists = controllerManagerPaths;
+          PathChanged = controllerManagerPaths;
+        };
+      };
+
       environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (!isNull cfg.etcClusterAdminKubeconfig)
         clusterAdminKubeconfig;
 

From 7323b77435f69362b0b4cc7edcb0915e9ab1ff48 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Wed, 6 Mar 2019 16:52:27 +0100
Subject: [PATCH 11/24] nixos/kubernetes: Address review: Separate preStart
 from certificates

---
 .../services/cluster/kubernetes/apiserver.nix |  5 +-
 .../cluster/kubernetes/controller-manager.nix |  5 +-
 .../services/cluster/kubernetes/flannel.nix   |  7 +++
 .../services/cluster/kubernetes/pki.nix       | 46 ++++++++++---------
 .../services/cluster/kubernetes/proxy.nix     |  5 +-
 .../services/cluster/kubernetes/scheduler.nix |  5 +-
 6 files changed, 39 insertions(+), 34 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix
index e4650c12cff9..72fb95358324 100644
--- a/nixos/modules/services/cluster/kubernetes/apiserver.nix
+++ b/nixos/modules/services/cluster/kubernetes/apiserver.nix
@@ -472,12 +472,11 @@ in
         after = [ "kube-scheduler.service" "kube-controller-manager.service" ];
         before = [ "kube-control-plane-online.target" ];
         preStart = ''
-          ${top.lib.mkWaitCurl (with top.pki.certs.flannelClient; {
+          ${top.lib.mkWaitCurl ( with config.systemd.services.kube-control-plane-online; {
             sleep = 3;
             path = "/healthz";
             cacert = top.caFile;
-            inherit cert key;
-          })}
+          } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
         '';
         script = "echo Ok";
         serviceConfig = {
diff --git a/nixos/modules/services/cluster/kubernetes/controller-manager.nix b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
index cc43a243df1c..a39fd62c689a 100644
--- a/nixos/modules/services/cluster/kubernetes/controller-manager.nix
+++ b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
@@ -111,12 +111,11 @@ in
       after = [ "kube-apiserver.service" ];
       before = [ "kube-control-plane-online.target" ];
       preStart = ''
-        ${top.lib.mkWaitCurl (with top.pki.certs.controllerManagerClient; {
+        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-controller-manager; {
           sleep = 1;
           path = "/api";
           cacert = top.caFile;
-          inherit cert key;
-        })}
+        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
       '';
       serviceConfig = {
         RestartSec = "30s";
diff --git a/nixos/modules/services/cluster/kubernetes/flannel.nix b/nixos/modules/services/cluster/kubernetes/flannel.nix
index fba70e3b9200..f85ebdafa316 100644
--- a/nixos/modules/services/cluster/kubernetes/flannel.nix
+++ b/nixos/modules/services/cluster/kubernetes/flannel.nix
@@ -80,6 +80,13 @@ in
       after = [ "kubelet.target" ];
       before = [ "flannel.target" ];
       path = [ pkgs.iptables ];
+      preStart = ''
+        ${top.lib.mkWaitCurl ( with config.systemd.services.flannel; {
+          path = "/api/v1/nodes";
+          cacert = top.caFile;
+          args = "-o - | grep podCIDR >/dev/null";
+        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+      '';
     };
 
     systemd.services.docker = {
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 98284fba12ac..14af3840eee3 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -344,6 +344,7 @@ in
       };
 
       systemd.services.kube-controller-manager = mkIf top.controllerManager.enable {
+        environment = { inherit (cfg.certs.controllerManagerClient) cert key; };
         unitConfig.ConditionPathExists = controllerManagerPaths;
       };
 
@@ -355,6 +356,25 @@ in
         };
       };
 
+      systemd.services.kube-scheduler = mkIf top.scheduler.enable {
+        environment = { inherit (top.pki.certs.schedulerClient) cert key; };
+        unitConfig.ConditionPathExists = schedulerPaths;
+      };
+
+      systemd.paths.kube-scheduler = mkIf top.scheduler.enable {
+        wantedBy = [ "kube-scheduler.service" ];
+        pathConfig = {
+          PathExists = schedulerPaths;
+          PathChanged = schedulerPaths;
+        };
+      };
+
+      systemd.services.kube-control-plane-online.environment = let
+        client = with cfg.certs; if top.apiserver.enable then clusterAdmin else kubelet;
+      in {
+        inherit (client) cert key;
+      };
+
       environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (!isNull cfg.etcClusterAdminKubeconfig)
         clusterAdminKubeconfig;
 
@@ -419,19 +439,12 @@ in
         };
       };
 
-      systemd.services.flannel = {
-        preStart = ''
-          ${top.lib.mkWaitCurl (with top.pki.certs.flannelClient; {
-            path = "/api/v1/nodes";
-            cacert = top.caFile;
-            inherit cert key;
-            args = "-o - | grep podCIDR >/dev/null";
-          })}
-        '';
+      systemd.services.flannel = mkIf top.flannel.enable {
+        environment = { inherit (top.pki.certs.flannelClient) cert key; };
         unitConfig.ConditionPathExists = flannelPaths;
       };
 
-      systemd.paths.flannel = {
+      systemd.paths.flannel = mkIf top.flannel.enable {
         wantedBy = [ "flannel.service" ];
         pathConfig = {
           PathExists = flannelPaths;
@@ -440,6 +453,7 @@ in
       };
 
       systemd.services.kube-proxy = mkIf top.proxy.enable {
+        environment = { inherit (top.pki.certs.kubeProxyClient) cert key; };
         unitConfig.ConditionPathExists = proxyPaths;
       };
 
@@ -451,18 +465,6 @@ in
         };
       };
 
-      systemd.services.kube-scheduler = mkIf top.scheduler.enable {
-        unitConfig.ConditionPathExists = schedulerPaths;
-      };
-
-      systemd.paths.kube-scheduler = mkIf top.scheduler.enable {
-        wantedBy = [ "kube-scheduler.service" ];
-        pathConfig = {
-          PathExists = schedulerPaths;
-          PathChanged = schedulerPaths;
-        };
-      };
-
       services.kubernetes = {
 
         apiserver = mkIf top.apiserver.enable (with cfg.certs.apiServer; {
diff --git a/nixos/modules/services/cluster/kubernetes/proxy.nix b/nixos/modules/services/cluster/kubernetes/proxy.nix
index d13d23e997b7..01d59e9ac883 100644
--- a/nixos/modules/services/cluster/kubernetes/proxy.nix
+++ b/nixos/modules/services/cluster/kubernetes/proxy.nix
@@ -53,11 +53,10 @@ in
       before = [ "node-online.target" ];
       path = with pkgs; [ iptables conntrack_tools ];
       preStart = ''
-        ${top.lib.mkWaitCurl (with top.pki.certs.kubeProxyClient; {
+        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-proxy; {
           path = "/api/v1/nodes/${top.kubelet.hostname}";
           cacert = top.caFile;
-          inherit cert key;
-        })}
+        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
       '';
       serviceConfig = {
         Slice = "kubernetes.slice";
diff --git a/nixos/modules/services/cluster/kubernetes/scheduler.nix b/nixos/modules/services/cluster/kubernetes/scheduler.nix
index 4aea9e9b6bd0..32a84563076b 100644
--- a/nixos/modules/services/cluster/kubernetes/scheduler.nix
+++ b/nixos/modules/services/cluster/kubernetes/scheduler.nix
@@ -63,12 +63,11 @@ in
       after = [ "kube-apiserver.service" ];
       before = [ "kube-control-plane-online.target" ];
       preStart = ''
-        ${top.lib.mkWaitCurl (with top.pki.certs.schedulerClient; {
+        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-scheduler; {
           sleep = 1;
           path = "/api";
           cacert = top.caFile;
-          inherit cert key;
-        })}
+        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
       '';
       serviceConfig = {
         Slice = "kubernetes.slice";

From 5684034693fc4d22ae212bdfbb1e33f930de1cc1 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Wed, 6 Mar 2019 16:53:33 +0100
Subject: [PATCH 12/24] nixos/kubernetes: Address review: Remove restart from
 certmgr bootstrap service

---
 nixos/modules/services/cluster/kubernetes/pki.nix | 2 --
 1 file changed, 2 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 14af3840eee3..0b43f2034c22 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -254,8 +254,6 @@ in
       ];
       serviceConfig = {
         TimeoutSec = "500";
-        RestartSec = "1s";
-        Restart = "on-failure";
       };
     };
 

From e148cb040b84a55229e097cb9c6b1af3c2a5484f Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Wed, 6 Mar 2019 17:17:20 +0100
Subject: [PATCH 13/24] nixos/kubernetes: Address review: rename node-online
 target

---
 nixos/modules/services/cluster/kubernetes/flannel.nix |  4 ++--
 nixos/modules/services/cluster/kubernetes/kubelet.nix | 10 +++++-----
 nixos/modules/services/cluster/kubernetes/proxy.nix   |  4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/flannel.nix b/nixos/modules/services/cluster/kubernetes/flannel.nix
index f85ebdafa316..4c5fe7559ebd 100644
--- a/nixos/modules/services/cluster/kubernetes/flannel.nix
+++ b/nixos/modules/services/cluster/kubernetes/flannel.nix
@@ -71,8 +71,8 @@ in
     };
 
     systemd.targets.flannel = {
-      wantedBy = [ "node-online.target" ];
-      before = [ "node-online.target" ];
+      wantedBy = [ "kube-node-online.target" ];
+      before = [ "kube-node-online.target" ];
     };
 
     systemd.services.flannel = {
diff --git a/nixos/modules/services/cluster/kubernetes/kubelet.nix b/nixos/modules/services/cluster/kubernetes/kubelet.nix
index c4ecb2417e29..01cdfccccf9f 100644
--- a/nixos/modules/services/cluster/kubernetes/kubelet.nix
+++ b/nixos/modules/services/cluster/kubernetes/kubelet.nix
@@ -340,9 +340,9 @@ in
       };
 
       systemd.services.kubelet-online = {
-        wantedBy = [ "node-online.target" ];
+        wantedBy = [ "kube-node-online.target" ];
         after = [ "flannel.target" "kubelet.target" ];
-        before = [ "node-online.target" ];
+        before = [ "kube-node-online.target" ];
         # it is complicated. flannel needs kubelet to run the pause container before
         # it discusses the node CIDR with apiserver and afterwards configures and restarts
         # dockerd. Until then prevent creating any pods because they have to be recreated anyway
@@ -407,11 +407,11 @@ in
 
     {
       systemd.targets.kubelet = {
-        wantedBy = [ "node-online.target" ];
-        before = [ "node-online.target" ];
+        wantedBy = [ "kube-node-online.target" ];
+        before = [ "kube-node-online.target" ];
       };
 
-      systemd.targets.node-online = {
+      systemd.targets.kube-node-online = {
         wantedBy = [ "kubernetes.target" ];
         before = [ "kubernetes.target" ];
       };
diff --git a/nixos/modules/services/cluster/kubernetes/proxy.nix b/nixos/modules/services/cluster/kubernetes/proxy.nix
index 01d59e9ac883..65d4f9ccbfcb 100644
--- a/nixos/modules/services/cluster/kubernetes/proxy.nix
+++ b/nixos/modules/services/cluster/kubernetes/proxy.nix
@@ -48,9 +48,9 @@ in
   config = mkIf cfg.enable {
     systemd.services.kube-proxy = {
       description = "Kubernetes Proxy Service";
-      wantedBy = [ "node-online.target" ];
+      wantedBy = [ "kube-node-online.target" ];
       after = [ "kubelet-online.service" ];
-      before = [ "node-online.target" ];
+      before = [ "kube-node-online.target" ];
       path = with pkgs; [ iptables conntrack_tools ];
       preStart = ''
         ${top.lib.mkWaitCurl ( with config.systemd.services.kube-proxy; {

From ff382c18c8f8e3eba1fc3ff331b7146bcb3af674 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Wed, 6 Mar 2019 17:56:28 +0100
Subject: [PATCH 14/24] nixos/kubernetes: Address review: Move remaining paths
 to pki

---
 .../services/cluster/kubernetes/apiserver.nix | 41 +-----------
 .../services/cluster/kubernetes/kubelet.nix   | 19 +-----
 .../services/cluster/kubernetes/pki.nix       | 62 ++++++++++++++++++-
 3 files changed, 64 insertions(+), 58 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix
index 72fb95358324..63b485c43b84 100644
--- a/nixos/modules/services/cluster/kubernetes/apiserver.nix
+++ b/nixos/modules/services/cluster/kubernetes/apiserver.nix
@@ -272,25 +272,7 @@ in
   ###### implementation
   config = mkMerge [
 
-    (mkIf cfg.enable (let
-      apiserverPaths = [
-        cfg.clientCaFile
-        cfg.etcd.caFile
-        cfg.etcd.certFile
-        cfg.etcd.keyFile
-        cfg.kubeletClientCaFile
-        cfg.kubeletClientCertFile
-        cfg.kubeletClientKeyFile
-        cfg.serviceAccountKeyFile
-        cfg.tlsCertFile
-        cfg.tlsKeyFile
-      ];
-      etcdPaths = [
-        config.services.etcd.certFile
-        config.services.etcd.keyFile
-        config.services.etcd.trustedCaFile
-      ];
-    in {
+    (mkIf cfg.enable {
         systemd.services.kube-apiserver = {
           description = "Kubernetes APIServer Service";
           wantedBy = [ "kube-control-plane-online.target" ];
@@ -360,25 +342,6 @@ in
             Restart = "on-failure";
             RestartSec = 5;
           };
-          unitConfig.ConditionPathExists = apiserverPaths;
-        };
-
-        systemd.paths.kube-apiserver = {
-          wantedBy = [ "kube-apiserver.service" ];
-          pathConfig = {
-            PathExists = apiserverPaths;
-            PathChanged = apiserverPaths;
-          };
-        };
-
-        systemd.services.etcd.unitConfig.ConditionPathExists = etcdPaths;
-
-        systemd.paths.etcd = {
-          wantedBy = [ "etcd.service" ];
-          pathConfig = {
-            PathExists = etcdPaths;
-            PathChanged = etcdPaths;
-          };
         };
 
         services.etcd = {
@@ -459,7 +422,7 @@ in
         };
       };
 
-    }))
+    })
     {
       systemd.targets.kube-control-plane-online = {
         wantedBy = [ "kubernetes.target" ];
diff --git a/nixos/modules/services/cluster/kubernetes/kubelet.nix b/nixos/modules/services/cluster/kubernetes/kubelet.nix
index 01cdfccccf9f..8eb212b41ece 100644
--- a/nixos/modules/services/cluster/kubernetes/kubelet.nix
+++ b/nixos/modules/services/cluster/kubernetes/kubelet.nix
@@ -241,13 +241,7 @@ in
 
   ###### implementation
   config = mkMerge [
-    (mkIf cfg.enable (let
-      kubeletPaths = [
-        cfg.clientCaFile
-        cfg.tlsCertFile
-        cfg.tlsKeyFile
-      ];
-    in {
+    (mkIf cfg.enable {
       services.kubernetes.kubelet.seedDockerImages = [infraContainer];
 
       systemd.services.kubelet = {
@@ -310,15 +304,6 @@ in
           '';
           WorkingDirectory = top.dataDir;
         };
-        unitConfig.ConditionPathExists = kubeletPaths;
-      };
-
-      systemd.paths.kubelet = {
-        wantedBy =  [ "kubelet.service" ];
-        pathConfig = {
-          PathExists = kubeletPaths;
-          PathChanged = kubeletPaths;
-        };
       };
 
       systemd.services.docker.before = [ "kubelet.service" ];
@@ -387,7 +372,7 @@ in
       };
 
       services.kubernetes.kubelet.kubeconfig.server = mkDefault top.apiserverAddress;
-    }))
+    })
 
     (mkIf (cfg.enable && cfg.manifests != {}) {
       environment.etc = mapAttrs' (name: manifest:
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 0b43f2034c22..8bacc07b0089 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -125,6 +125,23 @@ in
       top.caFile
       certmgrAPITokenPath
     ];
+    apiserverPaths = [
+      top.apiserver.clientCaFile
+      top.apiserver.etcd.caFile
+      top.apiserver.etcd.certFile
+      top.apiserver.etcd.keyFile
+      top.apiserver.kubeletClientCaFile
+      top.apiserver.kubeletClientCertFile
+      top.apiserver.kubeletClientKeyFile
+      top.apiserver.serviceAccountKeyFile
+      top.apiserver.tlsCertFile
+      top.apiserver.tlsKeyFile
+    ];
+    etcdPaths = [
+      config.services.etcd.certFile
+      config.services.etcd.keyFile
+      config.services.etcd.trustedCaFile
+    ];
     addonManagerPaths = mkIf top.addonManager.enable [
       cfg.certs.addonManager.cert
       cfg.certs.addonManager.key
@@ -150,6 +167,11 @@ in
       cfg.certs.controllerManagerClient.cert
       cfg.certs.controllerManagerClient.key
     ];
+    kubeletPaths = [
+      top.kubelet.clientCaFile
+      top.kubelet.tlsCertFile
+      top.kubelet.tlsKeyFile
+    ];
   in
   {
 
@@ -415,7 +437,7 @@ in
 
       # isolate etcd on loopback at the master node
       # easyCerts doesn't support multimaster clusters anyway atm.
-      services.etcd = with cfg.certs.etcd; {
+      services.etcd = mkIf top.apiserver.enable (with cfg.certs.etcd; {
         listenClientUrls = ["https://127.0.0.1:2379"];
         listenPeerUrls = ["https://127.0.0.1:2380"];
         advertiseClientUrls = ["https://etcd.local:2379"];
@@ -424,11 +446,35 @@ in
         certFile = mkDefault cert;
         keyFile = mkDefault key;
         trustedCaFile = mkDefault caCert;
-      };
+      });
       networking.extraHosts = mkIf (config.services.etcd.enable) ''
         127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local
       '';
 
+      systemd.services.kube-apiserver = mkIf top.apiserver.enable {
+        unitConfig.ConditionPathExists = apiserverPaths;
+      };
+
+      systemd.paths.kube-apiserver = mkIf top.apiserver.enable {
+        wantedBy = [ "kube-apiserver.service" ];
+        pathConfig = {
+          PathExists = apiserverPaths;
+          PathChanged = apiserverPaths;
+        };
+      };
+
+      systemd.services.etcd = mkIf top.apiserver.enable {
+        unitConfig.ConditionPathExists = etcdPaths;
+      };
+
+      systemd.paths.etcd = mkIf top.apiserver.enable {
+        wantedBy = [ "etcd.service" ];
+        pathConfig = {
+          PathExists = etcdPaths;
+          PathChanged = etcdPaths;
+        };
+      };
+
       services.flannel = with cfg.certs.flannelClient; {
         kubeconfig = top.lib.mkKubeConfig "flannel" {
           server = top.apiserverAddress;
@@ -455,6 +501,18 @@ in
         unitConfig.ConditionPathExists = proxyPaths;
       };
 
+      systemd.services.kubelet = mkIf top.kubelet.enable {
+        unitConfig.ConditionPathExists = kubeletPaths;
+      };
+
+      systemd.paths.kubelet = mkIf top.kubelet.enable {
+        wantedBy =  [ "kubelet.service" ];
+        pathConfig = {
+          PathExists = kubeletPaths;
+          PathChanged = kubeletPaths;
+        };
+      };
+
       systemd.paths.kube-proxy = mkIf top.proxy.enable {
         wantedBy = [ "kube-proxy.service" ];
         pathConfig = {

From 154356d820179a04c073ceadad0a4594ec18bf7d Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Thu, 7 Mar 2019 14:34:15 +0100
Subject: [PATCH 15/24] nixos/kubernetes: Fix kube-control-plane-online must
 not be present

outside kubernetes module.
---
 .../services/cluster/kubernetes/apiserver.nix | 23 -------------------
 .../services/cluster/kubernetes/default.nix   | 23 +++++++++++++++++++
 2 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix
index 63b485c43b84..287844074593 100644
--- a/nixos/modules/services/cluster/kubernetes/apiserver.nix
+++ b/nixos/modules/services/cluster/kubernetes/apiserver.nix
@@ -423,30 +423,7 @@ in
       };
 
     })
-    {
-      systemd.targets.kube-control-plane-online = {
-        wantedBy = [ "kubernetes.target" ];
-        before = [ "kubernetes.target" ];
-      };
 
-      systemd.services.kube-control-plane-online = rec {
-        description = "Kubernetes control plane is online";
-        wantedBy = [ "kube-control-plane-online.target" ];
-        after = [ "kube-scheduler.service" "kube-controller-manager.service" ];
-        before = [ "kube-control-plane-online.target" ];
-        preStart = ''
-          ${top.lib.mkWaitCurl ( with config.systemd.services.kube-control-plane-online; {
-            sleep = 3;
-            path = "/healthz";
-            cacert = top.caFile;
-          } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
-        '';
-        script = "echo Ok";
-        serviceConfig = {
-          TimeoutSec = "500";
-        };
-      };
-    }
   ];
 
 }
diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix
index f1f544afc4d1..6560cff63282 100644
--- a/nixos/modules/services/cluster/kubernetes/default.nix
+++ b/nixos/modules/services/cluster/kubernetes/default.nix
@@ -299,6 +299,29 @@ in {
       services.kubernetes.apiserverAddress = mkDefault ("https://${if cfg.apiserver.advertiseAddress != null
                           then cfg.apiserver.advertiseAddress
                           else "${cfg.masterAddress}:${toString cfg.apiserver.securePort}"}");
+
+      systemd.targets.kube-control-plane-online = {
+        wantedBy = [ "kubernetes.target" ];
+        before = [ "kubernetes.target" ];
+      };
+
+      systemd.services.kube-control-plane-online = rec {
+        description = "Kubernetes control plane is online";
+        wantedBy = [ "kube-control-plane-online.target" ];
+        after = [ "kube-scheduler.service" "kube-controller-manager.service" ];
+        before = [ "kube-control-plane-online.target" ];
+        preStart = ''
+          ${cfg.lib.mkWaitCurl ( with config.systemd.services.kube-control-plane-online; {
+            sleep = 3;
+            path = "/healthz";
+            cacert = cfg.caFile;
+          } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+        '';
+        script = "echo Ok";
+        serviceConfig = {
+          TimeoutSec = "500";
+        };
+      };
     })
   ];
 }

From ee9dd4386a061594ad69ff5a3a683f899f9f8c93 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Mon, 11 Mar 2019 10:44:24 +0100
Subject: [PATCH 16/24] Cleanup pki: addon-manager

---
 .../cluster/kubernetes/addon-manager.nix      | 77 +++++++++++++++----
 .../services/cluster/kubernetes/pki.nix       | 58 ++++----------
 2 files changed, 76 insertions(+), 59 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/addon-manager.nix b/nixos/modules/services/cluster/kubernetes/addon-manager.nix
index 406b20b0d8d8..b9a56811d2bd 100644
--- a/nixos/modules/services/cluster/kubernetes/addon-manager.nix
+++ b/nixos/modules/services/cluster/kubernetes/addon-manager.nix
@@ -63,24 +63,48 @@ in
     };
 
     enable = mkEnableOption "Whether to enable Kubernetes addon manager.";
+
+    kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes addon manager";
+    bootstrapAddonsKubeconfig = top.lib.mkKubeConfigOptions "Kubernetes addon manager bootstrap";
   };
 
   ###### implementation
-  config = mkIf cfg.enable {
+  config = let
+
+    addonManagerPaths = filter (a: a != null) [
+      cfg.kubeconfig.caFile
+      cfg.kubeconfig.certFile
+      cfg.kubeconfig.keyFile
+    ];
+    bootstrapAddonsPaths = filter (a: a != null) [
+      cfg.bootstrapAddonsKubeconfig.caFile
+      cfg.bootstrapAddonsKubeconfig.certFile
+      cfg.bootstrapAddonsKubeconfig.keyFile
+    ];
+
+  in mkIf cfg.enable {
     environment.etc."kubernetes/addons".source = "${addons}/";
 
+    #TODO: Get rid of kube-addon-manager in the future for the following reasons
+    # - it is basically just a shell script wrapped around kubectl
+    # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount
+    # - it is designed to be used with k8s system components only
+    # - it would be better with a more Nix-oriented way of managing addons
     systemd.services.kube-addon-manager = {
       description = "Kubernetes addon manager";
-      wantedBy = [ "kube-control-plane-online.target" ];
-      after = [ "kube-addon-manager-bootstrap.service" ];
-      before = [ "kube-control-plane-online.target" ];
-      environment.ADDON_PATH = "/etc/kubernetes/addons/";
-      path = [ pkgs.gawk ];
+      wantedBy = [ "kubernetes.target" ];
+      after = [ "kube-node-online.target" ];
+      before = [ "kubernetes.target" ];
+      environment = {
+        ADDON_PATH = "/etc/kubernetes/addons/";
+        KUBECONFIG = top.lib.mkKubeConfig "kube-addon-manager" cfg.kubeconfig;
+      };
+      path = with pkgs; [ gawk kubectl ];
       preStart = ''
-        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-addon-manager; {
-          path = "/api/v1/namespaces/kube-system/serviceaccounts/default";
-          cacert = top.caFile;
-        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+        until kubectl -n kube-system get serviceaccounts/default 2>/dev/null; do
+          echo kubectl -n kube-system get serviceaccounts/default: exit status $?
+          sleep 2
+        done
       '';
       serviceConfig = {
         Slice = "kubernetes.slice";
@@ -91,27 +115,52 @@ in
         Restart = "on-failure";
         RestartSec = 10;
       };
+      unitConfig.ConditionPathExists = addonManagerPaths;
     };
 
+    systemd.paths.kube-addon-manager = {
+      wantedBy = [ "kube-addon-manager.service" ];
+      pathConfig = {
+        PathExists = addonManagerPaths;
+        PathChanged = addonManagerPaths;
+      };
+    };
+
+    services.kubernetes.addonManager.kubeconfig.server = mkDefault top.apiserverAddress;
+
     systemd.services.kube-addon-manager-bootstrap = mkIf (top.apiserver.enable && top.addonManager.bootstrapAddons != {}) {
       wantedBy = [ "kube-control-plane-online.target" ];
       after = [ "kube-apiserver.service" ];
       before = [ "kube-control-plane-online.target" ];
       path = [ pkgs.kubectl ];
+      environment = {
+        KUBECONFIG = top.lib.mkKubeConfig "kube-addon-manager-bootstrap" cfg.bootstrapAddonsKubeconfig;
+      };
       preStart = with pkgs; let
         files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v))
           cfg.bootstrapAddons;
       in ''
-        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-addon-manager-bootstrap; {
-          path = "/api";
-          cacert = top.caFile;
-        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+        until kubectl auth can-i '*' '*' -q 2>/dev/null; do
+          echo kubectl auth can-i '*' '*': exit status $?
+          sleep 2
+        done
 
         kubectl apply -f ${concatStringsSep " \\\n -f " files}
       '';
       script = "echo Ok";
+      unitConfig.ConditionPathExists = bootstrapAddonsPaths;
     };
 
+    systemd.paths.kube-addon-manager-bootstrap = {
+      wantedBy = [ "kube-addon-manager-bootstrap.service" ];
+      pathConfig = {
+        PathExists = bootstrapAddonsPaths;
+        PathChanged = bootstrapAddonsPaths;
+      };
+    };
+
+    services.kubernetes.addonManager.bootstrapAddonsKubeconfig.server = mkDefault top.apiserverAddress;
+
     services.kubernetes.addonManager.bootstrapAddons = mkIf isRBACEnabled
     (let
       name = system:kube-addon-manager;
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 8bacc07b0089..90b40dd4c1f6 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -27,12 +27,11 @@ let
   certmgrAPITokenPath = "${top.secretsPath}/${cfsslAPITokenBaseName}";
   cfsslAPITokenLength = 32;
 
-  clusterAdminKubeconfig = with cfg.certs.clusterAdmin;
-    top.lib.mkKubeConfig "cluster-admin" {
-        server = top.apiserverAddress;
-        certFile = cert;
-        keyFile = key;
-    };
+  clusterAdminKubeconfig = with cfg.certs.clusterAdmin; {
+    server = top.apiserverAddress;
+    certFile = cert;
+    keyFile = key;
+  };
 
   remote = with config.services; "https://${kubernetes.masterAddress}:${toString cfssl.port}";
 in
@@ -142,12 +141,6 @@ in
       config.services.etcd.keyFile
       config.services.etcd.trustedCaFile
     ];
-    addonManagerPaths = mkIf top.addonManager.enable [
-      cfg.certs.addonManager.cert
-      cfg.certs.addonManager.key
-      cfg.certs.clusterAdmin.cert
-      cfg.certs.clusterAdmin.key
-    ];
     flannelPaths = [
       cfg.certs.flannelClient.cert
       cfg.certs.flannelClient.key
@@ -331,38 +324,6 @@ in
         };
       };
 
-      systemd.services.kube-addon-manager-bootstrap = mkIf (top.apiserver.enable && top.addonManager.bootstrapAddons != {}) {
-        environment = {
-          KUBECONFIG = clusterAdminKubeconfig;
-          inherit (cfg.certs.clusterAdmin) cert key;
-        };
-      };
-
-      #TODO: Get rid of kube-addon-manager in the future for the following reasons
-      # - it is basically just a shell script wrapped around kubectl
-      # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount
-      # - it is designed to be used with k8s system components only
-      # - it would be better with a more Nix-oriented way of managing addons
-      systemd.services.kube-addon-manager = mkIf top.addonManager.enable {
-        environment = with cfg.certs.addonManager; {
-          KUBECONFIG = top.lib.mkKubeConfig "kube-addon-manager" {
-              server = top.apiserverAddress;
-              certFile = cert;
-              keyFile = key;
-          };
-          inherit cert key;
-        };
-        unitConfig.ConditionPathExists = addonManagerPaths;
-      };
-
-      systemd.paths.kube-addon-manager = mkIf top.addonManager.enable {
-        wantedBy = [ "kube-addon-manager.service" ];
-        pathConfig = {
-          PathExists = addonManagerPaths;
-          PathChanged = addonManagerPaths;
-        };
-      };
-
       systemd.services.kube-controller-manager = mkIf top.controllerManager.enable {
         environment = { inherit (cfg.certs.controllerManagerClient) cert key; };
         unitConfig.ConditionPathExists = controllerManagerPaths;
@@ -396,7 +357,7 @@ in
       };
 
       environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (!isNull cfg.etcClusterAdminKubeconfig)
-        clusterAdminKubeconfig;
+        (top.lib.mkKubeConfig "cluster-admin" clusterAdminKubeconfig);
 
       environment.systemPackages = mkIf (top.kubelet.enable || top.proxy.enable) [
       (pkgs.writeScriptBin "nixos-kubernetes-node-join" ''
@@ -538,6 +499,13 @@ in
           kubeletClientCertFile = mkDefault cfg.certs.apiserverKubeletClient.cert;
           kubeletClientKeyFile = mkDefault cfg.certs.apiserverKubeletClient.key;
         });
+        addonManager = mkIf top.addonManager.enable {
+          kubeconfig = with cfg.certs.addonManager; {
+            certFile = mkDefault cert;
+            keyFile = mkDefault key;
+          };
+          bootstrapAddonsKubeconfig = clusterAdminKubeconfig;
+        };
         controllerManager = mkIf top.controllerManager.enable {
           serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.key;
           rootCaFile = cfg.certs.controllerManagerClient.caCert;

From 8ab50cb239e4aaeb88c372171a79f1fd874dfe50 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Mon, 11 Mar 2019 10:47:58 +0100
Subject: [PATCH 17/24] Cleanup pki: apiserver and etcd

---
 .../services/cluster/kubernetes/apiserver.nix | 43 ++++++++++++++++++-
 .../services/cluster/kubernetes/pki.nix       | 41 ------------------
 2 files changed, 42 insertions(+), 42 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix
index 287844074593..677738b4ec5d 100644
--- a/nixos/modules/services/cluster/kubernetes/apiserver.nix
+++ b/nixos/modules/services/cluster/kubernetes/apiserver.nix
@@ -272,7 +272,27 @@ in
   ###### implementation
   config = mkMerge [
 
-    (mkIf cfg.enable {
+    (let
+
+      apiserverPaths = filter (a: a != null) [
+        cfg.clientCaFile
+        cfg.etcd.caFile
+        cfg.etcd.certFile
+        cfg.etcd.keyFile
+        cfg.kubeletClientCaFile
+        cfg.kubeletClientCertFile
+        cfg.kubeletClientKeyFile
+        cfg.serviceAccountKeyFile
+        cfg.tlsCertFile
+        cfg.tlsKeyFile
+      ];
+      etcdPaths = filter (a: a != null) [
+        config.services.etcd.trustedCaFile
+        config.services.etcd.certFile
+        config.services.etcd.keyFile
+      ];
+
+    in mkIf cfg.enable {
         systemd.services.kube-apiserver = {
           description = "Kubernetes APIServer Service";
           wantedBy = [ "kube-control-plane-online.target" ];
@@ -342,6 +362,15 @@ in
             Restart = "on-failure";
             RestartSec = 5;
           };
+          unitConfig.ConditionPathExists = apiserverPaths;
+        };
+
+        systemd.paths.kube-apiserver = mkIf top.apiserver.enable {
+          wantedBy = [ "kube-apiserver.service" ];
+          pathConfig = {
+            PathExists = apiserverPaths;
+            PathChanged = apiserverPaths;
+          };
         };
 
         services.etcd = {
@@ -355,6 +384,18 @@ in
           initialAdvertisePeerUrls = mkDefault ["https://${top.masterAddress}:2380"];
         };
 
+        systemd.services.etcd = {
+          unitConfig.ConditionPathExists = etcdPaths;
+        };
+
+        systemd.paths.etcd = {
+          wantedBy = [ "etcd.service" ];
+          pathConfig = {
+            PathExists = etcdPaths;
+            PathChanged = etcdPaths;
+          };
+        };
+
         services.kubernetes.addonManager.bootstrapAddons = mkIf isRBACEnabled {
 
           apiserver-kubelet-api-admin-crb = {
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 90b40dd4c1f6..85e1fc9671c6 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -124,23 +124,6 @@ in
       top.caFile
       certmgrAPITokenPath
     ];
-    apiserverPaths = [
-      top.apiserver.clientCaFile
-      top.apiserver.etcd.caFile
-      top.apiserver.etcd.certFile
-      top.apiserver.etcd.keyFile
-      top.apiserver.kubeletClientCaFile
-      top.apiserver.kubeletClientCertFile
-      top.apiserver.kubeletClientKeyFile
-      top.apiserver.serviceAccountKeyFile
-      top.apiserver.tlsCertFile
-      top.apiserver.tlsKeyFile
-    ];
-    etcdPaths = [
-      config.services.etcd.certFile
-      config.services.etcd.keyFile
-      config.services.etcd.trustedCaFile
-    ];
     flannelPaths = [
       cfg.certs.flannelClient.cert
       cfg.certs.flannelClient.key
@@ -412,30 +395,6 @@ in
         127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local
       '';
 
-      systemd.services.kube-apiserver = mkIf top.apiserver.enable {
-        unitConfig.ConditionPathExists = apiserverPaths;
-      };
-
-      systemd.paths.kube-apiserver = mkIf top.apiserver.enable {
-        wantedBy = [ "kube-apiserver.service" ];
-        pathConfig = {
-          PathExists = apiserverPaths;
-          PathChanged = apiserverPaths;
-        };
-      };
-
-      systemd.services.etcd = mkIf top.apiserver.enable {
-        unitConfig.ConditionPathExists = etcdPaths;
-      };
-
-      systemd.paths.etcd = mkIf top.apiserver.enable {
-        wantedBy = [ "etcd.service" ];
-        pathConfig = {
-          PathExists = etcdPaths;
-          PathChanged = etcdPaths;
-        };
-      };
-
       services.flannel = with cfg.certs.flannelClient; {
         kubeconfig = top.lib.mkKubeConfig "flannel" {
           server = top.apiserverAddress;

From ce83dc2c52dae33330d868abd358166f9a4cb77a Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Mon, 11 Mar 2019 10:50:32 +0100
Subject: [PATCH 18/24] Cleanup pki: controller-manager

---
 .../cluster/kubernetes/controller-manager.nix | 39 ++++++++++++++-----
 .../services/cluster/kubernetes/pki.nix       | 20 ----------
 2 files changed, 30 insertions(+), 29 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/controller-manager.nix b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
index a39fd62c689a..a28679dbb9a9 100644
--- a/nixos/modules/services/cluster/kubernetes/controller-manager.nix
+++ b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
@@ -104,18 +104,30 @@ in
   };
 
   ###### implementation
-  config = mkIf cfg.enable {
-    systemd.services.kube-controller-manager = {
+  config = let
+
+    controllerManagerPaths = filter (a: a != null) [
+      cfg.kubeconfig.caFile
+      cfg.kubeconfig.certFile
+      cfg.kubeconfig.keyFile
+      cfg.rootCaFile
+      cfg.serviceAccountKeyFile
+      cfg.tlsCertFile
+      cfg.tlsKeyFile
+    ];
+
+  in mkIf cfg.enable {
+    systemd.services.kube-controller-manager = rec {
       description = "Kubernetes Controller Manager Service";
       wantedBy = [ "kube-control-plane-online.target" ];
       after = [ "kube-apiserver.service" ];
       before = [ "kube-control-plane-online.target" ];
+      environment.KUBECONFIG = top.lib.mkKubeConfig "kube-controller-manager" cfg.kubeconfig;
       preStart = ''
-        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-controller-manager; {
-          sleep = 1;
-          path = "/api";
-          cacert = top.caFile;
-        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+        until kubectl auth can-i get /api -q 2>/dev/null; do
+          echo kubectl auth can-i get /api: exit status $?
+          sleep 2
+        done
       '';
       serviceConfig = {
         RestartSec = "30s";
@@ -128,7 +140,7 @@ in
             "--cluster-cidr=${cfg.clusterCidr}"} \
           ${optionalString (cfg.featureGates != [])
             "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
-          --kubeconfig=${top.lib.mkKubeConfig "kube-controller-manager" cfg.kubeconfig} \
+          --kubeconfig=${environment.KUBECONFIG} \
           --leader-elect=${boolToString cfg.leaderElect} \
           ${optionalString (cfg.rootCaFile!=null)
             "--root-ca-file=${cfg.rootCaFile}"} \
@@ -149,7 +161,16 @@ in
         User = "kubernetes";
         Group = "kubernetes";
       };
-      path = top.path;
+      path = top.path ++ [ pkgs.kubectl ];
+      unitConfig.ConditionPathExists = controllerManagerPaths;
+    };
+
+    systemd.paths.kube-controller-manager = {
+      wantedBy = [ "kube-controller-manager.service" ];
+      pathConfig = {
+        PathExists = controllerManagerPaths;
+        PathChanged = controllerManagerPaths;
+      };
     };
 
     services.kubernetes.pki.certs = with top.lib; {
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 85e1fc9671c6..3c7af73e0b9a 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -136,13 +136,6 @@ in
       cfg.certs.schedulerClient.cert
       cfg.certs.schedulerClient.key
     ];
-    controllerManagerPaths = [
-      top.controllerManager.rootCaFile
-      top.controllerManager.tlsCertFile
-      top.controllerManager.tlsKeyFile
-      cfg.certs.controllerManagerClient.cert
-      cfg.certs.controllerManagerClient.key
-    ];
     kubeletPaths = [
       top.kubelet.clientCaFile
       top.kubelet.tlsCertFile
@@ -307,19 +300,6 @@ in
         };
       };
 
-      systemd.services.kube-controller-manager = mkIf top.controllerManager.enable {
-        environment = { inherit (cfg.certs.controllerManagerClient) cert key; };
-        unitConfig.ConditionPathExists = controllerManagerPaths;
-      };
-
-      systemd.paths.kube-controller-manager = mkIf top.controllerManager.enable {
-        wantedBy = [ "kube-controller-manager.service" ];
-        pathConfig = {
-          PathExists = controllerManagerPaths;
-          PathChanged = controllerManagerPaths;
-        };
-      };
-
       systemd.services.kube-scheduler = mkIf top.scheduler.enable {
         environment = { inherit (top.pki.certs.schedulerClient) cert key; };
         unitConfig.ConditionPathExists = schedulerPaths;

From ea6985ffc1af67fe11581c0e4c0a28e852f26c0b Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Mon, 11 Mar 2019 10:53:59 +0100
Subject: [PATCH 19/24] Cleanup pki: flannel

---
 .../services/cluster/kubernetes/flannel.nix   | 49 +++++++++++++++----
 .../services/cluster/kubernetes/pki.nix       | 31 +++---------
 2 files changed, 45 insertions(+), 35 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/flannel.nix b/nixos/modules/services/cluster/kubernetes/flannel.nix
index 4c5fe7559ebd..e79fbcb62002 100644
--- a/nixos/modules/services/cluster/kubernetes/flannel.nix
+++ b/nixos/modules/services/cluster/kubernetes/flannel.nix
@@ -24,16 +24,26 @@ in
   ###### interface
   options.services.kubernetes.flannel = {
     enable = mkEnableOption "enable flannel networking";
+    kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes flannel";
   };
 
   ###### implementation
-  config = mkIf cfg.enable {
+  config = let
+
+    flannelPaths = filter (a: a != null) [
+      cfg.kubeconfig.caFile
+      cfg.kubeconfig.certFile
+      cfg.kubeconfig.keyFile
+    ];
+    kubeconfig = top.lib.mkKubeConfig "flannel" cfg.kubeconfig;
+
+  in mkIf cfg.enable {
     services.flannel = {
 
       enable = mkDefault true;
       network = mkDefault top.clusterCidr;
-      inherit storageBackend;
-      nodeName = config.services.kubernetes.kubelet.hostname;
+      inherit storageBackend kubeconfig;
+      nodeName = top.kubelet.hostname;
     };
 
     services.kubernetes.kubelet = {
@@ -79,16 +89,35 @@ in
       wantedBy = [ "flannel.target" ];
       after = [ "kubelet.target" ];
       before = [ "flannel.target" ];
-      path = [ pkgs.iptables ];
-      preStart = ''
-        ${top.lib.mkWaitCurl ( with config.systemd.services.flannel; {
-          path = "/api/v1/nodes";
-          cacert = top.caFile;
-          args = "-o - | grep podCIDR >/dev/null";
-        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+      path = with pkgs; [ iptables kubectl ];
+      environment.KUBECONFIG = kubeconfig;
+      preStart = let
+        args = [
+          "--selector=kubernetes.io/hostname=${top.kubelet.hostname}"
+          # flannel exits if node is not registered yet, before that there is no podCIDR
+          "--output=jsonpath={.items[0].spec.podCIDR}"
+          # if jsonpath cannot be resolved exit with status 1
+          "--allow-missing-template-keys=false"
+        ];
+      in ''
+        until kubectl get nodes ${concatStringsSep " " args} 2>/dev/null; do
+          echo Waiting for ${top.kubelet.hostname} to be RegisteredNode
+          sleep 1
+        done
       '';
+      unitConfig.ConditionPathExists = flannelPaths;
     };
 
+    systemd.paths.flannel = {
+      wantedBy = [ "flannel.service" ];
+      pathConfig = {
+        PathExists = flannelPaths;
+        PathChanged = flannelPaths;
+      };
+    };
+
+    services.kubernetes.flannel.kubeconfig.server = mkDefault top.apiserverAddress;
+
     systemd.services.docker = {
       environment.DOCKER_OPTS = "-b none";
       serviceConfig.EnvironmentFile = "-/run/flannel/docker";
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 3c7af73e0b9a..2e79e7590e39 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -124,10 +124,6 @@ in
       top.caFile
       certmgrAPITokenPath
     ];
-    flannelPaths = [
-      cfg.certs.flannelClient.cert
-      cfg.certs.flannelClient.key
-    ];
     proxyPaths = mkIf top.proxy.enable [
       cfg.certs.kubeProxyClient.cert
       cfg.certs.kubeProxyClient.key
@@ -375,27 +371,6 @@ in
         127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local
       '';
 
-      services.flannel = with cfg.certs.flannelClient; {
-        kubeconfig = top.lib.mkKubeConfig "flannel" {
-          server = top.apiserverAddress;
-          certFile = cert;
-          keyFile = key;
-        };
-      };
-
-      systemd.services.flannel = mkIf top.flannel.enable {
-        environment = { inherit (top.pki.certs.flannelClient) cert key; };
-        unitConfig.ConditionPathExists = flannelPaths;
-      };
-
-      systemd.paths.flannel = mkIf top.flannel.enable {
-        wantedBy = [ "flannel.service" ];
-        pathConfig = {
-          PathExists = flannelPaths;
-          PathChanged = flannelPaths;
-        };
-      };
-
       systemd.services.kube-proxy = mkIf top.proxy.enable {
         environment = { inherit (top.pki.certs.kubeProxyClient) cert key; };
         unitConfig.ConditionPathExists = proxyPaths;
@@ -453,6 +428,12 @@ in
             keyFile = mkDefault key;
           };
         };
+        flannel = mkIf top.flannel.enable {
+          kubeconfig = with cfg.certs.flannelClient; {
+            certFile = cert;
+            keyFile = key;
+          };
+        };
         scheduler = mkIf top.scheduler.enable {
           kubeconfig = with cfg.certs.schedulerClient; {
             certFile = mkDefault cert;

From 73657b7fcfe8ad87af70a1b6186a355971da6c97 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Mon, 11 Mar 2019 10:58:57 +0100
Subject: [PATCH 20/24] Cleanup pki: kubelet

---
 .../services/cluster/kubernetes/kubelet.nix   | 25 +++++++++++++++++--
 .../services/cluster/kubernetes/pki.nix       | 17 -------------
 2 files changed, 23 insertions(+), 19 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/kubelet.nix b/nixos/modules/services/cluster/kubernetes/kubelet.nix
index 8eb212b41ece..2a4a0624555d 100644
--- a/nixos/modules/services/cluster/kubernetes/kubelet.nix
+++ b/nixos/modules/services/cluster/kubernetes/kubelet.nix
@@ -241,7 +241,18 @@ in
 
   ###### implementation
   config = mkMerge [
-    (mkIf cfg.enable {
+    (let
+
+      kubeletPaths = filter (a: a != null) [
+        cfg.kubeconfig.caFile
+        cfg.kubeconfig.certFile
+        cfg.kubeconfig.keyFile
+        cfg.clientCaFile
+        cfg.tlsCertFile
+        cfg.tlsKeyFile
+      ];
+
+    in mkIf cfg.enable {
       services.kubernetes.kubelet.seedDockerImages = [infraContainer];
 
       systemd.services.kubelet = {
@@ -304,6 +315,15 @@ in
           '';
           WorkingDirectory = top.dataDir;
         };
+        unitConfig.ConditionPathExists = kubeletPaths;
+      };
+
+      systemd.paths.kubelet = {
+        wantedBy =  [ "kubelet.service" ];
+        pathConfig = {
+          PathExists = kubeletPaths;
+          PathChanged = kubeletPaths;
+        };
       };
 
       systemd.services.docker.before = [ "kubelet.service" ];
@@ -321,6 +341,7 @@ in
         '';
         script = "echo Ok";
         serviceConfig.Type = "oneshot";
+        serviceConfig.RemainAfterExit = true;
         serviceConfig.Slice = "kubernetes.slice";
       };
 
@@ -337,7 +358,7 @@ in
           flannel-date = "stat --print=%Y ${docker-env}";
           docker-date = "systemctl show --property=ActiveEnterTimestamp --value docker";
         in ''
-          while ! test -f ${docker-env} ; do sleep 1 ; done
+          until test -f ${docker-env} ; do sleep 1 ; done
           while test `${flannel-date}` -gt `date +%s --date="$(${docker-date})"` ; do
             sleep 1
           done
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 2e79e7590e39..92eefae5bda4 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -132,11 +132,6 @@ in
       cfg.certs.schedulerClient.cert
       cfg.certs.schedulerClient.key
     ];
-    kubeletPaths = [
-      top.kubelet.clientCaFile
-      top.kubelet.tlsCertFile
-      top.kubelet.tlsKeyFile
-    ];
   in
   {
 
@@ -376,18 +371,6 @@ in
         unitConfig.ConditionPathExists = proxyPaths;
       };
 
-      systemd.services.kubelet = mkIf top.kubelet.enable {
-        unitConfig.ConditionPathExists = kubeletPaths;
-      };
-
-      systemd.paths.kubelet = mkIf top.kubelet.enable {
-        wantedBy =  [ "kubelet.service" ];
-        pathConfig = {
-          PathExists = kubeletPaths;
-          PathChanged = kubeletPaths;
-        };
-      };
-
       systemd.paths.kube-proxy = mkIf top.proxy.enable {
         wantedBy = [ "kube-proxy.service" ];
         pathConfig = {

From 46653f84c94ab9190a6841dbbfd874bb25d2a7f6 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Mon, 11 Mar 2019 11:01:54 +0100
Subject: [PATCH 21/24] Cleanup pki: proxy

---
 .../services/cluster/kubernetes/pki.nix       | 17 ----------
 .../services/cluster/kubernetes/proxy.nix     | 34 ++++++++++++++-----
 2 files changed, 26 insertions(+), 25 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 92eefae5bda4..be0b50e93299 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -124,10 +124,6 @@ in
       top.caFile
       certmgrAPITokenPath
     ];
-    proxyPaths = mkIf top.proxy.enable [
-      cfg.certs.kubeProxyClient.cert
-      cfg.certs.kubeProxyClient.key
-    ];
     schedulerPaths = mkIf top.scheduler.enable [
       cfg.certs.schedulerClient.cert
       cfg.certs.schedulerClient.key
@@ -366,19 +362,6 @@ in
         127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local
       '';
 
-      systemd.services.kube-proxy = mkIf top.proxy.enable {
-        environment = { inherit (top.pki.certs.kubeProxyClient) cert key; };
-        unitConfig.ConditionPathExists = proxyPaths;
-      };
-
-      systemd.paths.kube-proxy = mkIf top.proxy.enable {
-        wantedBy = [ "kube-proxy.service" ];
-        pathConfig = {
-          PathExists = proxyPaths;
-          PathChanged = proxyPaths;
-        };
-      };
-
       services.kubernetes = {
 
         apiserver = mkIf top.apiserver.enable (with cfg.certs.apiServer; {
diff --git a/nixos/modules/services/cluster/kubernetes/proxy.nix b/nixos/modules/services/cluster/kubernetes/proxy.nix
index 65d4f9ccbfcb..8a90542fe633 100644
--- a/nixos/modules/services/cluster/kubernetes/proxy.nix
+++ b/nixos/modules/services/cluster/kubernetes/proxy.nix
@@ -45,18 +45,27 @@ in
   };
 
   ###### implementation
-  config = mkIf cfg.enable {
-    systemd.services.kube-proxy = {
+  config = let
+
+    proxyPaths = filter (a: a != null) [
+      cfg.kubeconfig.caFile
+      cfg.kubeconfig.certFile
+      cfg.kubeconfig.keyFile
+    ];
+
+  in mkIf cfg.enable {
+    systemd.services.kube-proxy = rec {
       description = "Kubernetes Proxy Service";
       wantedBy = [ "kube-node-online.target" ];
       after = [ "kubelet-online.service" ];
       before = [ "kube-node-online.target" ];
-      path = with pkgs; [ iptables conntrack_tools ];
+      environment.KUBECONFIG = top.lib.mkKubeConfig "kube-proxy" cfg.kubeconfig;
+      path = with pkgs; [ iptables conntrack_tools kubectl ];
       preStart = ''
-        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-proxy; {
-          path = "/api/v1/nodes/${top.kubelet.hostname}";
-          cacert = top.caFile;
-        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+        until kubectl auth can-i get nodes/${top.kubelet.hostname} -q 2>/dev/null; do
+          echo kubectl auth can-i get nodes/${top.kubelet.hostname}: exit status $?
+          sleep 2
+        done
       '';
       serviceConfig = {
         Slice = "kubernetes.slice";
@@ -66,7 +75,7 @@ in
             "--cluster-cidr=${top.clusterCidr}"} \
           ${optionalString (cfg.featureGates != [])
             "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
-          --kubeconfig=${top.lib.mkKubeConfig "kube-proxy" cfg.kubeconfig} \
+          --kubeconfig=${environment.KUBECONFIG} \
           ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
           ${cfg.extraOpts}
         '';
@@ -74,6 +83,15 @@ in
         Restart = "on-failure";
         RestartSec = 5;
       };
+      unitConfig.ConditionPathExists = proxyPaths;
+    };
+
+    systemd.paths.kube-proxy = {
+      wantedBy = [ "kube-proxy.service" ];
+      pathConfig = {
+        PathExists = proxyPaths;
+        PathChanged = proxyPaths;
+      };
     };
 
     services.kubernetes.pki.certs = {

From 50c5f489ef4d9a1273860a5f5eaa9810f2c9d2ce Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Mon, 11 Mar 2019 11:03:40 +0100
Subject: [PATCH 22/24] Cleanup pki: scheduler

---
 .../services/cluster/kubernetes/pki.nix       | 17 ----------
 .../services/cluster/kubernetes/scheduler.nix | 34 ++++++++++++++-----
 2 files changed, 26 insertions(+), 25 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index be0b50e93299..6396ec229073 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -124,10 +124,6 @@ in
       top.caFile
       certmgrAPITokenPath
     ];
-    schedulerPaths = mkIf top.scheduler.enable [
-      cfg.certs.schedulerClient.cert
-      cfg.certs.schedulerClient.key
-    ];
   in
   {
 
@@ -287,19 +283,6 @@ in
         };
       };
 
-      systemd.services.kube-scheduler = mkIf top.scheduler.enable {
-        environment = { inherit (top.pki.certs.schedulerClient) cert key; };
-        unitConfig.ConditionPathExists = schedulerPaths;
-      };
-
-      systemd.paths.kube-scheduler = mkIf top.scheduler.enable {
-        wantedBy = [ "kube-scheduler.service" ];
-        pathConfig = {
-          PathExists = schedulerPaths;
-          PathChanged = schedulerPaths;
-        };
-      };
-
       systemd.services.kube-control-plane-online.environment = let
         client = with cfg.certs; if top.apiserver.enable then clusterAdmin else kubelet;
       in {
diff --git a/nixos/modules/services/cluster/kubernetes/scheduler.nix b/nixos/modules/services/cluster/kubernetes/scheduler.nix
index 32a84563076b..d58528259547 100644
--- a/nixos/modules/services/cluster/kubernetes/scheduler.nix
+++ b/nixos/modules/services/cluster/kubernetes/scheduler.nix
@@ -56,18 +56,27 @@ in
   };
 
   ###### implementation
-  config = mkIf cfg.enable {
-    systemd.services.kube-scheduler = {
+  config =  let
+
+    schedulerPaths = filter (a: a != null) [
+      cfg.kubeconfig.caFile
+      cfg.kubeconfig.certFile
+      cfg.kubeconfig.keyFile
+    ];
+
+  in mkIf cfg.enable {
+    systemd.services.kube-scheduler = rec {
       description = "Kubernetes Scheduler Service";
       wantedBy = [ "kube-control-plane-online.target" ];
       after = [ "kube-apiserver.service" ];
       before = [ "kube-control-plane-online.target" ];
+      environment.KUBECONFIG = top.lib.mkKubeConfig "kube-scheduler" cfg.kubeconfig;
+      path = [ pkgs.kubectl ];
       preStart = ''
-        ${top.lib.mkWaitCurl ( with config.systemd.services.kube-scheduler; {
-          sleep = 1;
-          path = "/api";
-          cacert = top.caFile;
-        } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
+        until kubectl auth can-i get /api -q 2>/dev/null; do
+          echo kubectl auth can-i get /api: exit status $?
+          sleep 2
+        done
       '';
       serviceConfig = {
         Slice = "kubernetes.slice";
@@ -75,7 +84,7 @@ in
           --address=${cfg.address} \
           ${optionalString (cfg.featureGates != [])
             "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
-          --kubeconfig=${top.lib.mkKubeConfig "kube-scheduler" cfg.kubeconfig} \
+          --kubeconfig=${environment.KUBECONFIG} \
           --leader-elect=${boolToString cfg.leaderElect} \
           --port=${toString cfg.port} \
           ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
@@ -87,6 +96,15 @@ in
         Restart = "on-failure";
         RestartSec = 5;
       };
+      unitConfig.ConditionPathExists = schedulerPaths;
+    };
+
+    systemd.paths.kube-scheduler = {
+      wantedBy = [ "kube-scheduler.service" ];
+      pathConfig = {
+        PathExists = schedulerPaths;
+        PathChanged = schedulerPaths;
+      };
     };
 
     services.kubernetes.pki.certs = {

From 45e683fbd6bc2b8ccf57b6425f4877deed618569 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Mon, 11 Mar 2019 11:42:12 +0100
Subject: [PATCH 23/24] Cleanup pki: control-plane-online

---
 .../services/cluster/kubernetes/default.nix   | 47 ++++++++++---------
 .../services/cluster/kubernetes/pki.nix       |  6 ---
 2 files changed, 25 insertions(+), 28 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix
index 6560cff63282..7cc172f12558 100644
--- a/nixos/modules/services/cluster/kubernetes/default.nix
+++ b/nixos/modules/services/cluster/kubernetes/default.nix
@@ -276,6 +276,30 @@ in {
         wantedBy = [ "multi-user.target" ];
       };
 
+      systemd.targets.kube-control-plane-online = {
+        wantedBy = [ "kubernetes.target" ];
+        before = [ "kubernetes.target" ];
+      };
+
+      systemd.services.kube-control-plane-online = rec {
+        description = "Kubernetes control plane is online";
+        wantedBy = [ "kube-control-plane-online.target" ];
+        after = [ "kube-scheduler.service" "kube-controller-manager.service" ];
+        before = [ "kube-control-plane-online.target" ];
+        environment.KUBECONFIG = cfg.lib.mkKubeConfig "default" cfg.kubeconfig;
+        path = [ pkgs.kubectl ];
+        preStart = ''
+          until kubectl get --raw=/healthz 2>/dev/null; do
+            echo kubectl get --raw=/healthz: exit status $?
+            sleep 3
+          done
+        '';
+        script = "echo Ok";
+        serviceConfig = {
+          TimeoutSec = "500";
+        };
+      };
+
       systemd.tmpfiles.rules = [
         "d /opt/cni/bin 0755 root root -"
         "d /run/kubernetes 0755 kubernetes kubernetes -"
@@ -300,28 +324,7 @@ in {
                           then cfg.apiserver.advertiseAddress
                           else "${cfg.masterAddress}:${toString cfg.apiserver.securePort}"}");
 
-      systemd.targets.kube-control-plane-online = {
-        wantedBy = [ "kubernetes.target" ];
-        before = [ "kubernetes.target" ];
-      };
-
-      systemd.services.kube-control-plane-online = rec {
-        description = "Kubernetes control plane is online";
-        wantedBy = [ "kube-control-plane-online.target" ];
-        after = [ "kube-scheduler.service" "kube-controller-manager.service" ];
-        before = [ "kube-control-plane-online.target" ];
-        preStart = ''
-          ${cfg.lib.mkWaitCurl ( with config.systemd.services.kube-control-plane-online; {
-            sleep = 3;
-            path = "/healthz";
-            cacert = cfg.caFile;
-          } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })}
-        '';
-        script = "echo Ok";
-        serviceConfig = {
-          TimeoutSec = "500";
-        };
-      };
+      services.kubernetes.kubeconfig.server = mkDefault cfg.apiserverAddress;
     })
   ];
 }
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 6396ec229073..1d0232fa2358 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -283,12 +283,6 @@ in
         };
       };
 
-      systemd.services.kube-control-plane-online.environment = let
-        client = with cfg.certs; if top.apiserver.enable then clusterAdmin else kubelet;
-      in {
-        inherit (client) cert key;
-      };
-
       environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (!isNull cfg.etcClusterAdminKubeconfig)
         (top.lib.mkKubeConfig "cluster-admin" clusterAdminKubeconfig);
 

From e3a80ebc40c9ce68db32a20dc806710b36393080 Mon Sep 17 00:00:00 2001
From: Christian Albrecht <christian.albrecht@mayflower.de>
Date: Mon, 11 Mar 2019 11:42:48 +0100
Subject: [PATCH 24/24] Cleanup pki: remove mkWaitCurl

---
 .../services/cluster/kubernetes/default.nix   | 13 -----------
 .../services/cluster/kubernetes/pki.nix       | 22 ++++++++++---------
 2 files changed, 12 insertions(+), 23 deletions(-)

diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix
index 7cc172f12558..84ed6821692a 100644
--- a/nixos/modules/services/cluster/kubernetes/default.nix
+++ b/nixos/modules/services/cluster/kubernetes/default.nix
@@ -73,18 +73,6 @@ let
     };
   };
 
-  mkWaitCurl = { address ? cfg.apiserverAddress, sleep ? 2, path ? "", args ? "-o /dev/null",
-                 cacert ? null, cert ? null, key ? null, }: ''
-    while ! ${pkgs.curl}/bin/curl --fail-early -fs \
-      ${if cacert != null then "--cacert ${cacert}" else ""} \
-      ${if cert != null then "--cert ${cert}" else ""} \
-      ${if key != null then "--key ${key}" else ""} \
-      ${address}${path} ${args} ; do
-        sleep ${toString sleep}
-        echo Waiting to be able to reach ${address}${path}
-    done
-  '';
-
   kubeConfigDefaults = {
     server = mkDefault cfg.kubeconfig.server;
     caFile = mkDefault cfg.kubeconfig.caFile;
@@ -174,7 +162,6 @@ in {
         inherit mkCert;
         inherit mkKubeConfig;
         inherit mkKubeConfigOptions;
-        inherit mkWaitCurl;
       };
       type = types.attrs;
     };
diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix
index 1d0232fa2358..4cf3269e18f3 100644
--- a/nixos/modules/services/cluster/kubernetes/pki.nix
+++ b/nixos/modules/services/cluster/kubernetes/pki.nix
@@ -182,12 +182,12 @@ in
       description = "Wait for ${remote} to be reachable.";
       wantedBy = [ "cfssl-online.target" ];
       before = [ "cfssl-online.target" ];
+      path = [ pkgs.curl ];
       preStart = ''
-        ${top.lib.mkWaitCurl {
-          address = remote;
-          path = "/api/v1/cfssl/info";
-          args = "-kd '{}' -o /dev/null";
-        }}
+        until curl --fail-early -fskd '{}' ${remote}/api/v1/cfssl/info -o /dev/null; do
+          echo curl ${remote}/api/v1/cfssl/info: exit status $?
+          sleep 2
+        done
       '';
       script = "echo Ok";
       serviceConfig = {
@@ -200,6 +200,7 @@ in
       wantedBy = [ "cfssl-online.target" ];
       after = [ "cfssl-online.target" ];
       before = [ "certmgr.service" ];
+      path = with pkgs; [ curl cfssl ];
       script = concatStringsSep "\n" [''
         set -e
 
@@ -218,11 +219,12 @@ in
       ''
       (optionalString (cfg.pkiTrustOnBootstrap) ''
         if [ ! -s "${top.caFile}" ]; then
-          ${top.lib.mkWaitCurl {
-            address = "https://${top.masterAddress}:${cfsslPort}";
-            path = "/api/v1/cfssl/info";
-            args = "-kd '{}' -o - | ${pkgs.cfssl}/bin/cfssljson -stdout >${top.caFile}";
-          }}
+          until test -s ${top.caFile}.json; do
+            sleep 2
+            curl --fail-early -fskd '{}' ${remote}/api/v1/cfssl/info -o ${top.caFile}.json
+          done
+          cfssljson -f ${top.caFile}.json -stdout >${top.caFile}
+          rm ${top.caFile}.json
         fi
       '')
       ];