forked from mirrors/nixpkgs
466beb0214
+ isolate etcd on the master node by letting it listen only on loopback + enabling kubelet on master and taint master with NoSchedule The reason for the latter is that flannel requires all nodes to be "registered" in the cluster in order to setup the cluster network. This means that the kubelet is needed even at nodes on which we don't plan to schedule anything.
389 lines
12 KiB
Nix
389 lines
12 KiB
Nix
{ config, lib, pkgs, ... }:
|
|
|
|
with lib;
|
|
|
|
let
|
|
top = config.services.kubernetes;
|
|
cfg = top.pki;
|
|
|
|
csrCA = pkgs.writeText "kube-pki-cacert-csr.json" (builtins.toJSON {
|
|
key = {
|
|
algo = "rsa";
|
|
size = 2048;
|
|
};
|
|
names = singleton cfg.caSpec;
|
|
});
|
|
|
|
csrCfssl = pkgs.writeText "kube-pki-cfssl-csr.json" (builtins.toJSON {
|
|
key = {
|
|
algo = "rsa";
|
|
size = 2048;
|
|
};
|
|
CN = top.masterAddress;
|
|
});
|
|
|
|
cfsslAPITokenBaseName = "apitoken.secret";
|
|
cfsslAPITokenPath = "${config.services.cfssl.dataDir}/${cfsslAPITokenBaseName}";
|
|
certmgrAPITokenPath = "${top.secretsPath}/${cfsslAPITokenBaseName}";
|
|
cfsslAPITokenLength = 32;
|
|
|
|
clusterAdminKubeconfig = with cfg.certs.clusterAdmin;
|
|
top.lib.mkKubeConfig "cluster-admin" {
|
|
server = top.apiserverAddress;
|
|
certFile = cert;
|
|
keyFile = key;
|
|
};
|
|
|
|
remote = with config.services; "https://${kubernetes.masterAddress}:${toString cfssl.port}";
|
|
in
|
|
{
|
|
###### interface
|
|
options.services.kubernetes.pki = with lib.types; {
|
|
|
|
enable = mkEnableOption "Whether to enable easyCert issuer service.";
|
|
|
|
certs = mkOption {
|
|
description = "List of certificate specs to feed to cert generator.";
|
|
default = {};
|
|
type = attrs;
|
|
};
|
|
|
|
genCfsslCACert = mkOption {
|
|
description = ''
|
|
Whether to automatically generate cfssl CA certificate and key,
|
|
if they don't exist.
|
|
'';
|
|
default = true;
|
|
type = bool;
|
|
};
|
|
|
|
genCfsslAPICerts = mkOption {
|
|
description = ''
|
|
Whether to automatically generate cfssl API webserver TLS cert and key,
|
|
if they don't exist.
|
|
'';
|
|
default = true;
|
|
type = bool;
|
|
};
|
|
|
|
genCfsslAPIToken = mkOption {
|
|
description = ''
|
|
Whether to automatically generate cfssl API-token secret,
|
|
if they doesn't exist.
|
|
'';
|
|
default = true;
|
|
type = bool;
|
|
};
|
|
|
|
pkiTrustOnBootstrap = mkOption {
|
|
description = "Whether to always trust remote cfssl server upon initial PKI bootstrap.";
|
|
default = true;
|
|
type = bool;
|
|
};
|
|
|
|
caCertPathPrefix = mkOption {
|
|
description = ''
|
|
Path-prefrix for the CA-certificate to be used for cfssl signing.
|
|
Suffixes ".pem" and "-key.pem" will be automatically appended for
|
|
the public and private keys respectively.
|
|
'';
|
|
default = "${config.services.cfssl.dataDir}/ca";
|
|
type = str;
|
|
};
|
|
|
|
caSpec = mkOption {
|
|
description = "Certificate specification for the auto-generated CAcert.";
|
|
default = {
|
|
CN = "kubernetes-cluster-ca";
|
|
O = "NixOS";
|
|
OU = "services.kubernetes.pki.caSpec";
|
|
L = "auto-generated";
|
|
};
|
|
type = attrs;
|
|
};
|
|
|
|
etcClusterAdminKubeconfig = mkOption {
|
|
description = ''
|
|
Symlink a kubeconfig with cluster-admin privileges to environment path
|
|
(/etc/<path>).
|
|
'';
|
|
default = null;
|
|
type = nullOr str;
|
|
};
|
|
|
|
};
|
|
|
|
###### implementation
|
|
config = mkIf cfg.enable
|
|
(let
|
|
cfsslCertPathPrefix = "${config.services.cfssl.dataDir}/cfssl";
|
|
cfsslCert = "${cfsslCertPathPrefix}.pem";
|
|
cfsslKey = "${cfsslCertPathPrefix}-key.pem";
|
|
in
|
|
{
|
|
|
|
services.cfssl = mkIf (top.apiserver.enable) {
|
|
enable = true;
|
|
address = "0.0.0.0";
|
|
tlsCert = cfsslCert;
|
|
tlsKey = cfsslKey;
|
|
configFile = toString (pkgs.writeText "cfssl-config.json" (builtins.toJSON {
|
|
signing = {
|
|
profiles = {
|
|
default = {
|
|
usages = ["digital signature"];
|
|
auth_key = "default";
|
|
expiry = "720h";
|
|
};
|
|
};
|
|
};
|
|
auth_keys = {
|
|
default = {
|
|
type = "standard";
|
|
key = "file:${cfsslAPITokenPath}";
|
|
};
|
|
};
|
|
}));
|
|
};
|
|
|
|
systemd.services.cfssl.preStart = with pkgs; with config.services.cfssl; mkIf (top.apiserver.enable)
|
|
(concatStringsSep "\n" [
|
|
"set -e"
|
|
(optionalString cfg.genCfsslCACert ''
|
|
if [ ! -f "${cfg.caCertPathPrefix}.pem" ]; then
|
|
${cfssl}/bin/cfssl genkey -initca ${csrCA} | \
|
|
${cfssl}/bin/cfssljson -bare ${cfg.caCertPathPrefix}
|
|
fi
|
|
'')
|
|
(optionalString cfg.genCfsslAPICerts ''
|
|
if [ ! -f "${dataDir}/cfssl.pem" ]; then
|
|
${cfssl}/bin/cfssl gencert -ca "${cfg.caCertPathPrefix}.pem" -ca-key "${cfg.caCertPathPrefix}-key.pem" ${csrCfssl} | \
|
|
${cfssl}/bin/cfssljson -bare ${cfsslCertPathPrefix}
|
|
fi
|
|
'')
|
|
(optionalString cfg.genCfsslAPIToken ''
|
|
if [ ! -f "${cfsslAPITokenPath}" ]; then
|
|
head -c ${toString (cfsslAPITokenLength / 2)} /dev/urandom | od -An -t x | tr -d ' ' >"${cfsslAPITokenPath}"
|
|
fi
|
|
chown cfssl "${cfsslAPITokenPath}" && chmod 400 "${cfsslAPITokenPath}"
|
|
'')]);
|
|
|
|
systemd.services.kube-certmgr-bootstrap = {
|
|
description = "Kubernetes certmgr bootstrapper";
|
|
wantedBy = [ "certmgr.service" ];
|
|
after = [ "cfssl.target" ];
|
|
script = concatStringsSep "\n" [''
|
|
set -e
|
|
|
|
# If there's a cfssl (cert issuer) running locally, then don't rely on user to
|
|
# manually paste it in place. Just symlink.
|
|
# otherwise, create the target file, ready for users to insert the token
|
|
|
|
if [ -f "${cfsslAPITokenPath}" ]; then
|
|
ln -fs "${cfsslAPITokenPath}" "${certmgrAPITokenPath}"
|
|
else
|
|
touch "${certmgrAPITokenPath}" && chmod 600 "${certmgrAPITokenPath}"
|
|
fi
|
|
''
|
|
(optionalString (cfg.pkiTrustOnBootstrap) ''
|
|
if [ ! -f "${top.caFile}" ] || [ $(cat "${top.caFile}" | wc -c) -lt 1 ]; then
|
|
${pkgs.curl}/bin/curl --fail-early -f -kd '{}' ${remote}/api/v1/cfssl/info | \
|
|
${pkgs.cfssl}/bin/cfssljson -stdout >${top.caFile}
|
|
fi
|
|
'')
|
|
];
|
|
serviceConfig = {
|
|
RestartSec = "10s";
|
|
Restart = "on-failure";
|
|
};
|
|
};
|
|
|
|
services.certmgr = {
|
|
enable = true;
|
|
package = pkgs.certmgr-selfsigned;
|
|
svcManager = "command";
|
|
specs =
|
|
let
|
|
mkSpec = _: cert: {
|
|
inherit (cert) action;
|
|
authority = {
|
|
inherit remote;
|
|
file.path = cert.caCert;
|
|
root_ca = cert.caCert;
|
|
profile = "default";
|
|
auth_key_file = certmgrAPITokenPath;
|
|
};
|
|
certificate = {
|
|
path = cert.cert;
|
|
};
|
|
private_key = cert.privateKeyOptions;
|
|
request = {
|
|
inherit (cert) CN hosts;
|
|
key = {
|
|
algo = "rsa";
|
|
size = 2048;
|
|
};
|
|
names = [ cert.fields ];
|
|
};
|
|
};
|
|
in
|
|
mapAttrs mkSpec cfg.certs;
|
|
};
|
|
|
|
#TODO: Get rid of kube-addon-manager in the future for the following reasons
|
|
# - it is basically just a shell script wrapped around kubectl
|
|
# - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount
|
|
# - it is designed to be used with k8s system components only
|
|
# - it would be better with a more Nix-oriented way of managing addons
|
|
systemd.services.kube-addon-manager = mkIf top.addonManager.enable (mkMerge [{
|
|
environment.KUBECONFIG = with cfg.certs.addonManager;
|
|
top.lib.mkKubeConfig "addon-manager" {
|
|
server = top.apiserverAddress;
|
|
certFile = cert;
|
|
keyFile = key;
|
|
};
|
|
}
|
|
|
|
(optionalAttrs (top.addonManager.bootstrapAddons != {}) {
|
|
serviceConfig.PermissionsStartOnly = true;
|
|
preStart = with pkgs;
|
|
let
|
|
files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v))
|
|
top.addonManager.bootstrapAddons;
|
|
in
|
|
''
|
|
export KUBECONFIG=${clusterAdminKubeconfig}
|
|
${kubectl}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files}
|
|
'';
|
|
})]);
|
|
|
|
environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (!isNull cfg.etcClusterAdminKubeconfig)
|
|
clusterAdminKubeconfig;
|
|
|
|
environment.systemPackages = mkIf (top.kubelet.enable || top.proxy.enable) [
|
|
(pkgs.writeScriptBin "nixos-kubernetes-node-join" ''
|
|
set -e
|
|
exec 1>&2
|
|
|
|
if [ $# -gt 0 ]; then
|
|
echo "Usage: $(basename $0)"
|
|
echo ""
|
|
echo "No args. Apitoken must be provided on stdin."
|
|
echo "To get the apitoken, execute: 'sudo cat ${certmgrAPITokenPath}' on the master node."
|
|
exit 1
|
|
fi
|
|
|
|
if [ $(id -u) != 0 ]; then
|
|
echo "Run as root please."
|
|
exit 1
|
|
fi
|
|
|
|
read -r token
|
|
if [ ''${#token} != ${toString cfsslAPITokenLength} ]; then
|
|
echo "Token must be of length ${toString cfsslAPITokenLength}."
|
|
exit 1
|
|
fi
|
|
|
|
echo $token > ${certmgrAPITokenPath}
|
|
chmod 600 ${certmgrAPITokenPath}
|
|
|
|
echo "Restarting certmgr..." >&1
|
|
systemctl restart certmgr
|
|
|
|
echo "Waiting for certs to appear..." >&1
|
|
|
|
${optionalString top.kubelet.enable ''
|
|
while [ ! -f ${cfg.certs.kubelet.cert} ]; do sleep 1; done
|
|
echo "Restarting kubelet..." >&1
|
|
systemctl restart kubelet
|
|
''}
|
|
|
|
${optionalString top.proxy.enable ''
|
|
while [ ! -f ${cfg.certs.kubeProxyClient.cert} ]; do sleep 1; done
|
|
echo "Restarting kube-proxy..." >&1
|
|
systemctl restart kube-proxy
|
|
''}
|
|
|
|
${optionalString top.flannel.enable ''
|
|
while [ ! -f ${cfg.certs.flannelClient.cert} ]; do sleep 1; done
|
|
echo "Restarting flannel..." >&1
|
|
systemctl restart flannel
|
|
''}
|
|
|
|
echo "Node joined succesfully"
|
|
'')];
|
|
|
|
# isolate etcd on loopback at the master node
|
|
# easyCerts doesn't support multimaster clusters anyway atm.
|
|
services.etcd = with cfg.certs.etcd; {
|
|
listenClientUrls = ["https://127.0.0.1:2379"];
|
|
listenPeerUrls = ["https://127.0.0.1:2380"];
|
|
advertiseClientUrls = ["https://etcd.local:2379"];
|
|
initialCluster = ["${top.masterAddress}=https://etcd.local:2380"];
|
|
initialAdvertisePeerUrls = ["https://etcd.local:2380"];
|
|
certFile = mkDefault cert;
|
|
keyFile = mkDefault key;
|
|
trustedCaFile = mkDefault caCert;
|
|
};
|
|
networking.extraHosts = mkIf (config.services.etcd.enable) ''
|
|
127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local
|
|
'';
|
|
|
|
services.flannel = with cfg.certs.flannelClient; {
|
|
kubeconfig = top.lib.mkKubeConfig "flannel" {
|
|
server = top.apiserverAddress;
|
|
certFile = cert;
|
|
keyFile = key;
|
|
};
|
|
};
|
|
|
|
services.kubernetes = {
|
|
|
|
apiserver = mkIf top.apiserver.enable (with cfg.certs.apiServer; {
|
|
etcd = with cfg.certs.apiserverEtcdClient; {
|
|
servers = ["https://etcd.local:2379"];
|
|
certFile = mkDefault cert;
|
|
keyFile = mkDefault key;
|
|
caFile = mkDefault caCert;
|
|
};
|
|
clientCaFile = mkDefault caCert;
|
|
tlsCertFile = mkDefault cert;
|
|
tlsKeyFile = mkDefault key;
|
|
serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.cert;
|
|
kubeletClientCaFile = mkDefault caCert;
|
|
kubeletClientCertFile = mkDefault cfg.certs.apiserverKubeletClient.cert;
|
|
kubeletClientKeyFile = mkDefault cfg.certs.apiserverKubeletClient.key;
|
|
});
|
|
controllerManager = mkIf top.controllerManager.enable {
|
|
serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.key;
|
|
rootCaFile = cfg.certs.controllerManagerClient.caCert;
|
|
kubeconfig = with cfg.certs.controllerManagerClient; {
|
|
certFile = mkDefault cert;
|
|
keyFile = mkDefault key;
|
|
};
|
|
};
|
|
scheduler = mkIf top.scheduler.enable {
|
|
kubeconfig = with cfg.certs.schedulerClient; {
|
|
certFile = mkDefault cert;
|
|
keyFile = mkDefault key;
|
|
};
|
|
};
|
|
kubelet = mkIf top.kubelet.enable {
|
|
clientCaFile = mkDefault cfg.certs.kubelet.caCert;
|
|
tlsCertFile = mkDefault cfg.certs.kubelet.cert;
|
|
tlsKeyFile = mkDefault cfg.certs.kubelet.key;
|
|
kubeconfig = with cfg.certs.kubeletClient; {
|
|
certFile = mkDefault cert;
|
|
keyFile = mkDefault key;
|
|
};
|
|
};
|
|
proxy = mkIf top.proxy.enable {
|
|
kubeconfig = with cfg.certs.kubeProxyClient; {
|
|
certFile = mkDefault cert;
|
|
keyFile = mkDefault key;
|
|
};
|
|
};
|
|
};
|
|
});
|
|
}
|