From 6e9037fed0ce0b55ef37188ec1a58e18e196a780 Mon Sep 17 00:00:00 2001 From: Christian Albrecht Date: Wed, 6 Mar 2019 16:44:38 +0100 Subject: [PATCH] nixos/kubernetes: Address review: Move bootstrapping addons into own service --- .../cluster/kubernetes/addon-manager.nix | 26 +++++ .../services/cluster/kubernetes/flannel.nix | 102 ++++++------------ .../services/cluster/kubernetes/pki.nix | 46 +++----- 3 files changed, 78 insertions(+), 96 deletions(-) diff --git a/nixos/modules/services/cluster/kubernetes/addon-manager.nix b/nixos/modules/services/cluster/kubernetes/addon-manager.nix index 46f5b68b2a5c..406b20b0d8d8 100644 --- a/nixos/modules/services/cluster/kubernetes/addon-manager.nix +++ b/nixos/modules/services/cluster/kubernetes/addon-manager.nix @@ -72,9 +72,16 @@ in systemd.services.kube-addon-manager = { description = "Kubernetes addon manager"; wantedBy = [ "kube-control-plane-online.target" ]; + after = [ "kube-addon-manager-bootstrap.service" ]; before = [ "kube-control-plane-online.target" ]; environment.ADDON_PATH = "/etc/kubernetes/addons/"; path = [ pkgs.gawk ]; + preStart = '' + ${top.lib.mkWaitCurl ( with config.systemd.services.kube-addon-manager; { + path = "/api/v1/namespaces/kube-system/serviceaccounts/default"; + cacert = top.caFile; + } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })} + ''; serviceConfig = { Slice = "kubernetes.slice"; ExecStart = "${top.package}/bin/kube-addons"; @@ -86,6 +93,25 @@ in }; }; + systemd.services.kube-addon-manager-bootstrap = mkIf (top.apiserver.enable && top.addonManager.bootstrapAddons != {}) { + wantedBy = [ "kube-control-plane-online.target" ]; + after = [ "kube-apiserver.service" ]; + before = [ "kube-control-plane-online.target" ]; + path = [ pkgs.kubectl ]; + preStart = with pkgs; let + files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v)) + cfg.bootstrapAddons; + in '' + ${top.lib.mkWaitCurl ( with config.systemd.services.kube-addon-manager-bootstrap; { + path = "/api"; + cacert = top.caFile; + } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })} + + kubectl apply -f ${concatStringsSep " \\\n -f " files} + ''; + script = "echo Ok"; + }; + services.kubernetes.addonManager.bootstrapAddons = mkIf isRBACEnabled (let name = system:kube-addon-manager; diff --git a/nixos/modules/services/cluster/kubernetes/flannel.nix b/nixos/modules/services/cluster/kubernetes/flannel.nix index a5b4f7103dcb..fba70e3b9200 100644 --- a/nixos/modules/services/cluster/kubernetes/flannel.nix +++ b/nixos/modules/services/cluster/kubernetes/flannel.nix @@ -27,12 +27,7 @@ in }; ###### implementation - config = mkIf cfg.enable (let - flannelBootstrapPaths = mkIf top.apiserver.enable [ - top.pki.certs.clusterAdmin.cert - top.pki.certs.clusterAdmin.key - ]; - in { + config = mkIf cfg.enable { services.flannel = { enable = mkDefault true; @@ -112,69 +107,42 @@ in }; # give flannel som kubernetes rbac permissions if applicable - systemd.services.flannel-rbac-bootstrap = mkIf (top.apiserver.enable && (elem "RBAC" top.apiserver.authorizationMode)) { + services.kubernetes.addonManager.bootstrapAddons = mkIf ((storageBackend == "kubernetes") && (elem "RBAC" top.apiserver.authorizationMode)) { + flannel-cr = { + apiVersion = "rbac.authorization.k8s.io/v1beta1"; + kind = "ClusterRole"; + metadata = { name = "flannel"; }; + rules = [{ + apiGroups = [ "" ]; + resources = [ "pods" ]; + verbs = [ "get" ]; + } + { + apiGroups = [ "" ]; + resources = [ "nodes" ]; + verbs = [ "list" "watch" ]; + } + { + apiGroups = [ "" ]; + resources = [ "nodes/status" ]; + verbs = [ "patch" ]; + }]; + }; - wantedBy = [ "kube-apiserver-online.target" ]; - after = [ "kube-apiserver-online.target" ]; - before = [ "flannel.service" ]; - path = with pkgs; [ kubectl ]; - preStart = let - files = mapAttrsToList (n: v: pkgs.writeText "${n}.json" (builtins.toJSON v)) { - flannel-cr = { - apiVersion = "rbac.authorization.k8s.io/v1beta1"; - kind = "ClusterRole"; - metadata = { name = "flannel"; }; - rules = [{ - apiGroups = [ "" ]; - resources = [ "pods" ]; - verbs = [ "get" ]; - } - { - apiGroups = [ "" ]; - resources = [ "nodes" ]; - verbs = [ "list" "watch" ]; - } - { - apiGroups = [ "" ]; - resources = [ "nodes/status" ]; - verbs = [ "patch" ]; - }]; - }; - - flannel-crb = { - apiVersion = "rbac.authorization.k8s.io/v1beta1"; - kind = "ClusterRoleBinding"; - metadata = { name = "flannel"; }; - roleRef = { - apiGroup = "rbac.authorization.k8s.io"; - kind = "ClusterRole"; - name = "flannel"; - }; - subjects = [{ - kind = "User"; - name = "flannel-client"; - }]; - }; + flannel-crb = { + apiVersion = "rbac.authorization.k8s.io/v1beta1"; + kind = "ClusterRoleBinding"; + metadata = { name = "flannel"; }; + roleRef = { + apiGroup = "rbac.authorization.k8s.io"; + kind = "ClusterRole"; + name = "flannel"; }; - in '' - ${top.lib.mkWaitCurl (with top.pki.certs.clusterAdmin; { - path = "/"; - cacert = top.caFile; - inherit cert key; - })} - - kubectl -s ${top.apiserverAddress} --certificate-authority=${top.caFile} --client-certificate=${top.pki.certs.clusterAdmin.cert} --client-key=${top.pki.certs.clusterAdmin.key} apply -f ${concatStringsSep " \\\n -f " files} - ''; - script = "echo Ok"; - unitConfig.ConditionPathExists = flannelBootstrapPaths; - }; - - systemd.paths.flannel-rbac-bootstrap = mkIf top.apiserver.enable { - wantedBy = [ "flannel-rbac-bootstrap.service" ]; - pathConfig = { - PathExists = flannelBootstrapPaths; - PathChanged = flannelBootstrapPaths; + subjects = [{ + kind = "User"; + name = "flannel-client"; + }]; }; }; - }); + }; } diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix index 329278e375ca..4d97d8322cd4 100644 --- a/nixos/modules/services/cluster/kubernetes/pki.nix +++ b/nixos/modules/services/cluster/kubernetes/pki.nix @@ -304,41 +304,29 @@ in }; }; + systemd.services.kube-addon-manager-bootstrap = mkIf (top.apiserver.enable && top.addonManager.bootstrapAddons != {}) { + environment = { + KUBECONFIG = clusterAdminKubeconfig; + inherit (cfg.certs.clusterAdmin) cert key; + }; + }; + #TODO: Get rid of kube-addon-manager in the future for the following reasons # - it is basically just a shell script wrapped around kubectl # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount # - it is designed to be used with k8s system components only # - it would be better with a more Nix-oriented way of managing addons - systemd.services.kube-addon-manager = mkIf top.addonManager.enable (mkMerge [{ - environment.KUBECONFIG = with cfg.certs.addonManager; - top.lib.mkKubeConfig "addon-manager" { - server = top.apiserverAddress; - certFile = cert; - keyFile = key; + systemd.services.kube-addon-manager = mkIf top.addonManager.enable { + environment = with cfg.certs.addonManager; { + KUBECONFIG = top.lib.mkKubeConfig "kube-addon-manager" { + server = top.apiserverAddress; + certFile = cert; + keyFile = key; }; - } - - (optionalAttrs (top.addonManager.bootstrapAddons != {}) { - serviceConfig.PermissionsStartOnly = true; - preStart = with pkgs; - let - files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v)) - top.addonManager.bootstrapAddons; - in - '' - export KUBECONFIG=${clusterAdminKubeconfig} - ${kubectl}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files} - - ${top.lib.mkWaitCurl (with top.pki.certs.addonManager; { - path = "/api/v1/namespaces/kube-system/serviceaccounts/default"; - cacert = top.caFile; - inherit cert key; - })} - ''; - }) - { - unitConfig.ConditionPathExists = addonManagerPaths; - }]); + inherit cert key; + }; + unitConfig.ConditionPathExists = addonManagerPaths; + }; systemd.paths.kube-addon-manager = mkIf top.addonManager.enable { wantedBy = [ "kube-addon-manager.service" ];