nixpkgs/nixos/modules/services/cluster/kubernetes/flannel.nix

{ config, lib, pkgs, ... }:

with lib;

let
  top = config.services.kubernetes;
  cfg = top.flannel;

  # we want flannel to use kubernetes itself as configuration backend, not direct etcd
  storageBackend = "kubernetes";

  # needed for flannel to pass options to docker
  mkDockerOpts = pkgs.runCommand "mk-docker-opts" {
    buildInputs = [ pkgs.makeWrapper ];
  } ''
    mkdir -p $out
    cp ${pkgs.kubernetes.src}/cluster/centos/node/bin/mk-docker-opts.sh $out/mk-docker-opts.sh

    # bashInteractive needed for `compgen`
    makeWrapper ${pkgs.bashInteractive}/bin/bash $out/mk-docker-opts --add-flags "$out/mk-docker-opts.sh"
  '';
in
{
  ###### interface
  options.services.kubernetes.flannel = {
    enable = mkEnableOption "enable flannel networking";
  };

  ###### implementation
  config = mkIf cfg.enable (let
    flannelBootstrapPaths = mkIf top.apiserver.enable [
      top.pki.certs.clusterAdmin.cert
      top.pki.certs.clusterAdmin.key
    ];
  in {
    services.flannel = {

      enable = mkDefault true;
      network = mkDefault top.clusterCidr;
      inherit storageBackend;
      nodeName = config.services.kubernetes.kubelet.hostname;
    };

    services.kubernetes.kubelet = {
      networkPlugin = mkDefault "cni";
      cni.config = mkDefault [{
        name = "mynet";
        type = "flannel";
        delegate = {
          isDefaultGateway = true;
          bridge = "docker0";
        };
      }];
    };

    systemd.services.mk-docker-opts = {
      description = "Pre-Docker Actions";
      wantedBy = [ "flannel.target" ];
      before = [ "flannel.target" ];
      path = with pkgs; [ gawk gnugrep ];
      script = ''
        ${mkDockerOpts}/mk-docker-opts -d /run/flannel/docker
        systemctl restart docker
      '';
      unitConfig.ConditionPathExists = [ "/run/flannel/subnet.env" ];
      serviceConfig.Type = "oneshot";
    };

    systemd.paths.flannel-subnet-env = {
      wantedBy = [ "mk-docker-opts.service" ];
      pathConfig = {
        PathExists = [ "/run/flannel/subnet.env" ];
        PathChanged = [ "/run/flannel/subnet.env" ];
        Unit = "mk-docker-opts.service";
      };
    };

    systemd.targets.flannel = {
      wantedBy = [ "node-online.target" ];
      before = [ "node-online.target" ];
    };

    systemd.services.flannel = {
      wantedBy = [ "flannel.target" ];
      after = [ "kubelet.target" ];
      before = [ "flannel.target" ];
    };

    systemd.services.docker = {
      environment.DOCKER_OPTS = "-b none";
      serviceConfig.EnvironmentFile = "-/run/flannel/docker";
    };

    # read environment variables generated by mk-docker-opts
    virtualisation.docker.extraOptions = "$DOCKER_OPTS";

    networking = {
      firewall.allowedUDPPorts = [
        8285  # flannel udp
        8472  # flannel vxlan
      ];
      dhcpcd.denyInterfaces = [ "docker*" "flannel*" ];
    };

    services.kubernetes.pki.certs = {
      flannelClient = top.lib.mkCert {
        name = "flannel-client";
        CN = "flannel-client";
        action = "systemctl restart flannel.service";
      };
    };

    # give flannel som kubernetes rbac permissions if applicable
    systemd.services.flannel-rbac-bootstrap = mkIf (top.apiserver.enable && (elem "RBAC" top.apiserver.authorizationMode)) {

      wantedBy = [ "kube-apiserver-online.target" ];
      after = [ "kube-apiserver-online.target" ];
      before = [ "flannel.service" ];
      path = with pkgs; [ kubectl ];
      preStart = let
        files = mapAttrsToList (n: v: pkgs.writeText "${n}.json" (builtins.toJSON v)) {
          flannel-cr = {
            apiVersion = "rbac.authorization.k8s.io/v1beta1";
            kind = "ClusterRole";
            metadata = { name = "flannel"; };
            rules = [{
              apiGroups = [ "" ];
              resources = [ "pods" ];
              verbs = [ "get" ];
            }
            {
              apiGroups = [ "" ];
              resources = [ "nodes" ];
              verbs = [ "list" "watch" ];
            }
            {
              apiGroups = [ "" ];
              resources = [ "nodes/status" ];
              verbs = [ "patch" ];
            }];
          };

          flannel-crb = {
            apiVersion = "rbac.authorization.k8s.io/v1beta1";
            kind = "ClusterRoleBinding";
            metadata = { name = "flannel"; };
            roleRef = {
              apiGroup = "rbac.authorization.k8s.io";
              kind = "ClusterRole";
              name = "flannel";
            };
            subjects = [{
              kind = "User";
              name = "flannel-client";
            }];
          };
        };
      in ''
        ${top.lib.mkWaitCurl (with top.pki.certs.clusterAdmin; {
          path = "/";
          cacert = top.caFile;
          inherit cert key;
        })}

        kubectl -s ${top.apiserverAddress} --certificate-authority=${top.caFile} --client-certificate=${top.pki.certs.clusterAdmin.cert} --client-key=${top.pki.certs.clusterAdmin.key} apply -f ${concatStringsSep " \\\n -f " files}
      '';
      script = "echo Ok";
      unitConfig.ConditionPathExists = flannelBootstrapPaths;
    };

    systemd.paths.flannel-rbac-bootstrap = mkIf top.apiserver.enable {
      wantedBy = [ "flannel-rbac-bootstrap.service" ];
      pathConfig = {
        PathExists = flannelBootstrapPaths;
        PathChanged = flannelBootstrapPaths;
      };
    };
  });
}