From a4d72ad6289a81b73e000617f9c9efc7f1d17499 Mon Sep 17 00:00:00 2001 From: Jappie Klooster Date: Wed, 18 May 2022 09:41:54 -0400 Subject: [PATCH] nixos/keter: init Enable keter module Keter is an apploader which: 1. has the old app running on a port. 2. loads a new one, and wait for that to complete 3. switches the old with the new one once the new one finished loading. It supports more functionality but this use case is the primary one being used by supercede. Adds keter as a module to nixos. Currently keter is unusable with nix, because it relies on bundeling of a tar and uploading that to a specific folder. These expressions automate these devops tasks, with especially nixops in mind. This will work with versions above 1.8 The test seems to work. This uses a new version of keter which has good support for status code on error pages. We're using this config at production at supercede so it should be fine. Squash log: ========== mention keter in changelog Update generated release notes Always restart keter on failure This is a little bit of extra stability in case keter crashes. Which can happen under extreme conditions (DoS attacks). Update nixos/doc/manual/release-notes/rl-2205.section.md Co-authored-by: ckie <25263210+ckiee@users.noreply.github.com> Update nixos/modules/module-list.nix Co-authored-by: ckie <25263210+ckiee@users.noreply.github.com> Remove sanitization don't put domain in as a string Update nixos/tests/keter.nix Co-authored-by: ckie <25263210+ckiee@users.noreply.github.com> add jappie as module maintainer Use type path instead of two seperate options Fix generated docs added test machinery to figure out why it's failing Fix the test, use console output run nixpkgs-fmt on all modules Inline config file. This get's rid of a lot of inderection as well. Run nix format remove comment simplify executable for test delete config file add config for keter root Remove after redis clause set keter root by default to /var/lib/keter Update nixos/modules/services/web-servers/keter/default.nix Co-authored-by: ckie <25263210+ckiee@users.noreply.github.com> Update nixos/modules/services/web-servers/keter/default.nix Co-authored-by: ckie <25263210+ckiee@users.noreply.github.com> Update nixos/modules/services/web-servers/keter/default.nix Co-authored-by: ckie <25263210+ckiee@users.noreply.github.com> fix nit add newlines add default text and move description in a long description Delete rather obvious comment fix release db thing remove longDescription and put it in a comment instead change description of mkEnalbeOption explain what keter does by using the hackage synopsis set domain to keterDomain and same for executable move comment to where it's happening fix type error add formatting better comment try add seperate user for keter Revert "try add seperate user for keter" This reverts commit d3522d36c96117335bfa072e6f453406c244e940. Doing this breaks the setup set default to avoid needing cap_net_bind_service remove weird comment use example fields eleborated on process leakage Update nixos/modules/services/web-servers/keter/default.nix Co-authored-by: ckie <25263210+ckiee@users.noreply.github.com> run nixpkgs-fmt update docs Fix formatting, set keter package by default format our little nixexpr replace '' -> " where possible drop indent for multiline string make description much shorter regen docs database --- .../from_md/release-notes/rl-2205.section.xml | 7 + .../manual/release-notes/rl-2205.section.md | 1 + nixos/modules/module-list.nix | 1 + .../services/web-servers/keter/bundle.nix | 40 +++++ .../services/web-servers/keter/default.nix | 162 ++++++++++++++++++ nixos/tests/all-tests.nix | 1 + nixos/tests/keter.nix | 42 +++++ 7 files changed, 254 insertions(+) create mode 100644 nixos/modules/services/web-servers/keter/bundle.nix create mode 100644 nixos/modules/services/web-servers/keter/default.nix create mode 100644 nixos/tests/keter.nix diff --git a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml index 5208671e4dab..3dc4a494588b 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml @@ -2122,6 +2122,13 @@ sudo mkdir /var/lib/redis-peertube sudo cp /var/lib/redis/dump.rdb /var/lib/redis-peertube/dump.rdb + + + Added the keter NixOS module. Keter reverse + proxies requests to your loaded application based on virtual + hostnames. + + If you are using Wayland you can choose to use the Ozone diff --git a/nixos/doc/manual/release-notes/rl-2205.section.md b/nixos/doc/manual/release-notes/rl-2205.section.md index faf941f56996..f1f544035002 100644 --- a/nixos/doc/manual/release-notes/rl-2205.section.md +++ b/nixos/doc/manual/release-notes/rl-2205.section.md @@ -776,6 +776,7 @@ In addition to numerous new and upgraded packages, this release has the followin sudo mkdir /var/lib/redis-peertube sudo cp /var/lib/redis/dump.rdb /var/lib/redis-peertube/dump.rdb ``` +- Added the `keter` NixOS module. Keter reverse proxies requests to your loaded application based on virtual hostnames. - If you are using Wayland you can choose to use the Ozone Wayland support in Chrome and several Electron apps by setting the environment variable diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index d67602a26761..08834022cf46 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -1118,6 +1118,7 @@ ./services/web-servers/pomerium.nix ./services/web-servers/unit/default.nix ./services/web-servers/tomcat.nix + ./services/web-servers/keter ./services/web-servers/traefik.nix ./services/web-servers/trafficserver/default.nix ./services/web-servers/ttyd.nix diff --git a/nixos/modules/services/web-servers/keter/bundle.nix b/nixos/modules/services/web-servers/keter/bundle.nix new file mode 100644 index 000000000000..32b08c3be206 --- /dev/null +++ b/nixos/modules/services/web-servers/keter/bundle.nix @@ -0,0 +1,40 @@ +/* This makes a keter bundle as described on the github page: + https://github.com/snoyberg/keter#bundling-your-app-for-keter +*/ +{ keterDomain +, keterExecutable +, gnutar +, writeTextFile +, lib +, stdenv +, ... +}: + +let + str.stanzas = [{ + # we just use nix as an absolute path so we're not bundling any binaries + type = "webapp"; + /* Note that we're not actually putting the executable in the bundle, + we already can use the nix store for copying, so we just + symlink to the app. */ + exec = keterExecutable; + host = keterDomain; + }]; + configFile = writeTextFile { + name = "keter.yml"; + text = (lib.generators.toYAML { } str); + }; + +in +stdenv.mkDerivation { + name = "keter-bundle"; + buildCommand = '' + mkdir -p config + cp ${configFile} config/keter.yaml + + echo 'create a gzipped tarball' + mkdir -p $out + tar -zcvf $out/bundle.tar.gz.keter ./. + ''; + buildInputs = [ gnutar ]; +} diff --git a/nixos/modules/services/web-servers/keter/default.nix b/nixos/modules/services/web-servers/keter/default.nix new file mode 100644 index 000000000000..83e221add37e --- /dev/null +++ b/nixos/modules/services/web-servers/keter/default.nix @@ -0,0 +1,162 @@ +{ config, pkgs, lib, ... }: +let + cfg = config.services.keter; +in +{ + meta = { + maintainers = with lib.maintainers; [ jappie ]; + }; + + options.services.keter = { + enable = lib.mkEnableOption ''keter, a web app deployment manager. +Note that this module only support loading of webapps: +Keep an old app running and swap the ports when the new one is booted. +''; + + keterRoot = lib.mkOption { + type = lib.types.str; + default = "/var/lib/keter"; + description = "Mutable state folder for keter"; + }; + + keterPackage = lib.mkOption { + type = lib.types.package; + default = pkgs.haskellPackages.keter; + defaultText = lib.literalExpression "pkgs.haskellPackages.keter"; + description = "The keter package to be used"; + }; + + globalKeterConfig = lib.mkOption { + type = lib.types.attrs; + default = { + ip-from-header = true; + listeners = [{ + host = "*4"; + port = 6981; + }]; + }; + # You want that ip-from-header in the nginx setup case + # so it's not set to 127.0.0.1. + # using a port above 1024 allows you to avoid needing CAP_NET_BIND_SERVICE + defaultText = lib.literalExpression '' + { + ip-from-header = true; + listeners = [{ + host = "*4"; + port = 6981; + }]; + } + ''; + description = "Global config for keter"; + }; + + bundle = { + appName = lib.mkOption { + type = lib.types.str; + default = "myapp"; + description = "The name keter assigns to this bundle"; + }; + + executable = lib.mkOption { + type = lib.types.path; + description = "The executable to be run"; + }; + + domain = lib.mkOption { + type = lib.types.str; + default = "example.com"; + description = "The domain keter will bind to"; + }; + + publicScript = lib.mkOption { + type = lib.types.str; + default = ""; + description = '' + Allows loading of public environment variables, + these are emitted to the log so it shouldn't contain secrets. + ''; + example = "ADMIN_EMAIL=hi@example.com"; + }; + + secretScript = lib.mkOption { + type = lib.types.str; + default = ""; + description = "Allows loading of private environment variables"; + example = "MY_AWS_KEY=$(cat /run/keys/AWS_ACCESS_KEY_ID)"; + }; + }; + + }; + + config = lib.mkIf cfg.enable ( + let + incoming = "${cfg.keterRoot}/incoming"; + + + globalKeterConfigFile = pkgs.writeTextFile { + name = "keter-config.yml"; + text = (lib.generators.toYAML { } (cfg.globalKeterConfig // { root = cfg.keterRoot; })); + }; + + # If things are expected to change often, put it in the bundle! + bundle = pkgs.callPackage ./bundle.nix + (cfg.bundle // { keterExecutable = executable; keterDomain = cfg.bundle.domain; }); + + # This indirection is required to ensure the nix path + # gets copied over to the target machine in remote deployments. + # Furthermore, it's important that we use exec to + # run the binary otherwise we get process leakage due to this + # being executed on every change. + executable = pkgs.writeShellScript "bundle-wrapper" '' + set -e + ${cfg.bundle.secretScript} + set -xe + ${cfg.bundle.publicScript} + exec ${cfg.bundle.executable} + ''; + + in + { + systemd.services.keter = { + description = "keter app loader"; + script = '' + set -xe + mkdir -p ${incoming} + { tail -F ${cfg.keterRoot}/log/keter/current.log -n 0 & ${cfg.keterPackage}/bin/keter ${globalKeterConfigFile}; } + ''; + wantedBy = [ "multi-user.target" "nginx.service" ]; + + serviceConfig = { + Restart = "always"; + RestartSec = "10s"; + }; + + after = [ + "network.target" + "local-fs.target" + "postgresql.service" + ]; + }; + + # On deploy this will load our app, by moving it into the incoming dir + # If the bundle content changes, this will run again. + # Because the bundle content contains the nix path to the exectuable, + # we inherit nix based cache busting. + systemd.services.load-keter-bundle = { + description = "load keter bundle into incoming folder"; + after = [ "keter.service" ]; + wantedBy = [ "multi-user.target" ]; + # we can't override keter bundles because it'll stop the previous app + # https://github.com/snoyberg/keter#deploying + script = '' + set -xe + cp ${bundle}/bundle.tar.gz.keter ${incoming}/${cfg.bundle.appName}.keter + ''; + path = [ + executable + cfg.bundle.executable + ]; # this is a hack to get the executable copied over to the machine. + }; + } + ); +} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 1064d62da930..e6fd508100dd 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -264,6 +264,7 @@ in { kerberos = handleTest ./kerberos/default.nix {}; kernel-generic = handleTest ./kernel-generic.nix {}; kernel-latest-ath-user-regd = handleTest ./kernel-latest-ath-user-regd.nix {}; + keter = handleTest ./keter.nix {}; kexec = handleTest ./kexec.nix {}; keycloak = discoverTests (import ./keycloak.nix); keymap = handleTest ./keymap.nix {}; diff --git a/nixos/tests/keter.nix b/nixos/tests/keter.nix new file mode 100644 index 000000000000..0bfb96e1c324 --- /dev/null +++ b/nixos/tests/keter.nix @@ -0,0 +1,42 @@ +import ./make-test-python.nix ({ pkgs, ... }: +let + port = 81; +in +{ + name = "keter"; + meta = with pkgs.lib.maintainers; { + maintainers = [ jappie ]; + }; + + + nodes.machine = { config, pkgs, ... }: { + services.keter = { + enable = true; + + globalKeterConfig = { + listeners = [{ + host = "*4"; + inherit port; + }]; + }; + bundle = { + appName = "test-bundle"; + domain = "localhost"; + executable = pkgs.writeShellScript "run" '' + ${pkgs.python3}/bin/python -m http.server $PORT + ''; + }; + }; + }; + + testScript = + '' + machine.wait_for_unit("keter.service") + + machine.wait_for_open_port(${toString port}) + machine.wait_for_console_text("Activating app test-bundle with hosts: localhost") + + + machine.succeed("curl --fail http://localhost:${toString port}/") + ''; +})