From bcdc4d976cfc2ab8a922f716ea90048b1328b371 Mon Sep 17 00:00:00 2001 From: Minijackson Date: Mon, 31 Jul 2023 14:38:26 +0200 Subject: [PATCH 1/6] nixos/netbox: remove "with lib;" --- nixos/modules/services/web-apps/netbox.nix | 52 +++++++++++----------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/nixos/modules/services/web-apps/netbox.nix b/nixos/modules/services/web-apps/netbox.nix index e2ef350ba4e5..8d4a26e6ace6 100644 --- a/nixos/modules/services/web-apps/netbox.nix +++ b/nixos/modules/services/web-apps/netbox.nix @@ -1,7 +1,5 @@ { config, lib, pkgs, ... }: -with lib; - let cfg = config.services.netbox; pythonFmt = pkgs.formats.pythonVars {}; @@ -17,7 +15,7 @@ let pkg = (cfg.package.overrideAttrs (old: { installPhase = old.installPhase + '' ln -s ${configFile} $out/opt/netbox/netbox/netbox/configuration.py - '' + optionalString cfg.enableLdap '' + '' + lib.optionalString cfg.enableLdap '' ln -s ${cfg.ldapConfigPath} $out/opt/netbox/netbox/netbox/ldap_config.py ''; })).override { @@ -31,7 +29,7 @@ let in { options.services.netbox = { - enable = mkOption { + enable = lib.mkOption { type = lib.types.bool; default = false; description = lib.mdDoc '' @@ -66,18 +64,18 @@ in { }; }; - listenAddress = mkOption { - type = types.str; + listenAddress = lib.mkOption { + type = lib.types.str; default = "[::1]"; description = lib.mdDoc '' Address the server will listen on. ''; }; - package = mkOption { - type = types.package; - default = if versionAtLeast config.system.stateVersion "23.05" then pkgs.netbox else pkgs.netbox_3_3; - defaultText = literalExpression '' + package = lib.mkOption { + type = lib.types.package; + default = if lib.versionAtLeast config.system.stateVersion "23.05" then pkgs.netbox else pkgs.netbox_3_3; + defaultText = lib.literalExpression '' if versionAtLeast config.system.stateVersion "23.05" then pkgs.netbox else pkgs.netbox_3_3; ''; description = lib.mdDoc '' @@ -85,18 +83,18 @@ in { ''; }; - port = mkOption { - type = types.port; + port = lib.mkOption { + type = lib.types.port; default = 8001; description = lib.mdDoc '' Port the server will listen on. ''; }; - plugins = mkOption { - type = types.functionTo (types.listOf types.package); + plugins = lib.mkOption { + type = with lib.types; functionTo (listOf package); default = _: []; - defaultText = literalExpression '' + defaultText = lib.literalExpression '' python3Packages: with python3Packages; []; ''; description = lib.mdDoc '' @@ -104,23 +102,23 @@ in { ''; }; - dataDir = mkOption { - type = types.str; + dataDir = lib.mkOption { + type = lib.types.str; default = "/var/lib/netbox"; description = lib.mdDoc '' Storage path of netbox. ''; }; - secretKeyFile = mkOption { - type = types.path; + secretKeyFile = lib.mkOption { + type = lib.types.path; description = lib.mdDoc '' Path to a file containing the secret key. ''; }; - extraConfig = mkOption { - type = types.lines; + extraConfig = lib.mkOption { + type = lib.types.lines; default = ""; description = lib.mdDoc '' Additional lines of configuration appended to the `configuration.py`. @@ -128,8 +126,8 @@ in { ''; }; - enableLdap = mkOption { - type = types.bool; + enableLdap = lib.mkOption { + type = lib.types.bool; default = false; description = lib.mdDoc '' Enable LDAP-Authentication for Netbox. @@ -138,8 +136,8 @@ in { ''; }; - ldapConfigPath = mkOption { - type = types.path; + ldapConfigPath = lib.mkOption { + type = lib.types.path; default = ""; description = lib.mdDoc '' Path to the Configuration-File for LDAP-Authentication, will be loaded as `ldap_config.py`. @@ -173,9 +171,9 @@ in { }; }; - config = mkIf cfg.enable { + config = lib.mkIf cfg.enable { services.netbox = { - plugins = mkIf cfg.enableLdap (ps: [ ps.django-auth-ldap ]); + plugins = lib.mkIf cfg.enableLdap (ps: [ ps.django-auth-ldap ]); settings = { STATIC_ROOT = staticDir; MEDIA_ROOT = "${cfg.dataDir}/media"; From a57a322b8dfe6be69f3567469596682edf0ef73d Mon Sep 17 00:00:00 2001 From: Minijackson Date: Mon, 31 Jul 2023 14:39:08 +0200 Subject: [PATCH 2/6] nixos/netbox: add GIT_PATH environment variable used by the synchronization backend for remote git repositories, in "Data Sources" --- nixos/modules/services/web-apps/netbox.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nixos/modules/services/web-apps/netbox.nix b/nixos/modules/services/web-apps/netbox.nix index 8d4a26e6ace6..67bbf1073423 100644 --- a/nixos/modules/services/web-apps/netbox.nix +++ b/nixos/modules/services/web-apps/netbox.nix @@ -180,6 +180,8 @@ in { REPORTS_ROOT = "${cfg.dataDir}/reports"; SCRIPTS_ROOT = "${cfg.dataDir}/scripts"; + GIT_PATH = "${pkgs.gitMinimal}/bin/git"; + DATABASE = { NAME = "netbox"; USER = "netbox"; From d1b0a9543db7f64f6d443a0e8629e86ad8d68e16 Mon Sep 17 00:00:00 2001 From: Minijackson Date: Mon, 31 Jul 2023 14:50:09 +0200 Subject: [PATCH 3/6] nixos/netbox: move migration into the preStart netbox.service, reindex Now migrations are run only on upgrade / downgrade and first start, which makes netbox much faster on a normal start. add the reindex for NetBox > 3.5.0, to populate the index, preventing empty search results. Migrations were moved out of netbox-migration.service into netbox.service, to prevent service dependency issues when upgrading NixOS. --- nixos/modules/services/web-apps/netbox.nix | 44 +++++++++++----------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/nixos/modules/services/web-apps/netbox.nix b/nixos/modules/services/web-apps/netbox.nix index 67bbf1073423..5f42f42a9af9 100644 --- a/nixos/modules/services/web-apps/netbox.nix +++ b/nixos/modules/services/web-apps/netbox.nix @@ -264,39 +264,39 @@ in { RestartSec = 30; }; in { - netbox-migration = { - description = "NetBox migrations"; - wantedBy = [ "netbox.target" ]; - - environment = { - PYTHONPATH = pkg.pythonPath; - }; - - serviceConfig = defaultServiceConfig // { - Type = "oneshot"; - ExecStart = '' - ${pkg}/bin/netbox migrate - ''; - PrivateTmp = true; - }; - }; - netbox = { description = "NetBox WSGI Service"; documentation = [ "https://docs.netbox.dev/" ]; wantedBy = [ "netbox.target" ]; - after = [ "network-online.target" "netbox-migration.service" ]; + after = [ "network-online.target" ]; wants = [ "network-online.target" ]; + environment.PYTHONPATH = pkg.pythonPath; + preStart = '' + # On the first run, or on upgrade / downgrade, run migrations and related. + # This mostly correspond to upstream NetBox's 'upgrade.sh' script. + versionFile="${cfg.dataDir}/version" + + if [[ -e "$versionFile" && "$(cat "$versionFile")" == "${cfg.package.version}" ]]; then + exit 0 + fi + + ${pkg}/bin/netbox migrate ${pkg}/bin/netbox trace_paths --no-input ${pkg}/bin/netbox collectstatic --no-input ${pkg}/bin/netbox remove_stale_contenttypes --no-input - ''; + # TODO: remove the condition when we remove netbox_3_3 + ${lib.optionalString + (lib.versionAtLeast cfg.package.version "3.5.0") + "${pkg}/bin/netbox reindex --lazy"} + ${pkg}/bin/netbox clearsessions + ${pkg}/bin/netbox clearcache - environment.PYTHONPATH = pkg.pythonPath; + echo "${cfg.package.version}" > "$versionFile" + ''; serviceConfig = defaultServiceConfig // { ExecStart = '' @@ -331,7 +331,7 @@ in { wantedBy = [ "multi-user.target" ]; - after = [ "network-online.target" ]; + after = [ "network-online.target" "netbox.service" ]; wants = [ "network-online.target" ]; environment.PYTHONPATH = pkg.pythonPath; @@ -351,7 +351,7 @@ in { wantedBy = [ "multi-user.target" ]; - after = [ "network-online.target" ]; + after = [ "network-online.target" "netbox.service" ]; wants = [ "network-online.target" ]; timerConfig = { From de8086be4f91bb6e6aa5be83fa3adb5a3b45fd79 Mon Sep 17 00:00:00 2001 From: Minijackson Date: Mon, 31 Jul 2023 14:56:05 +0200 Subject: [PATCH 4/6] nixos/tests/netbox-upgrade: init Test that the upgrade from NetBox 3.3 to NetBox 3.5 runs fine --- nixos/tests/all-tests.nix | 1 + nixos/tests/web-apps/netbox-upgrade.nix | 85 +++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 nixos/tests/web-apps/netbox-upgrade.nix diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 9ab0bfb21f4a..b4a2512bfa1c 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -519,6 +519,7 @@ in { networking.scripted = handleTest ./networking.nix { networkd = false; }; netbox = handleTest ./web-apps/netbox.nix { inherit (pkgs) netbox; }; netbox_3_3 = handleTest ./web-apps/netbox.nix { netbox = pkgs.netbox_3_3; }; + netbox-upgrade = handleTest ./web-apps/netbox-upgrade.nix {}; # TODO: put in networking.nix after the test becomes more complete networkingProxy = handleTest ./networking-proxy.nix {}; nextcloud = handleTest ./nextcloud {}; diff --git a/nixos/tests/web-apps/netbox-upgrade.nix b/nixos/tests/web-apps/netbox-upgrade.nix new file mode 100644 index 000000000000..602cf8d889d4 --- /dev/null +++ b/nixos/tests/web-apps/netbox-upgrade.nix @@ -0,0 +1,85 @@ +import ../make-test-python.nix ({ lib, pkgs, ... }: let + oldNetbox = pkgs.netbox_3_3; +in { + name = "netbox-upgrade"; + + meta = with lib.maintainers; { + maintainers = [ minijackson ]; + }; + + nodes.machine = { config, ... }: { + services.netbox = { + enable = true; + package = oldNetbox; + secretKeyFile = pkgs.writeText "secret" '' + abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 + ''; + }; + + services.nginx = { + enable = true; + + recommendedProxySettings = true; + + virtualHosts.netbox = { + default = true; + locations."/".proxyPass = "http://localhost:${toString config.services.netbox.port}"; + locations."/static/".alias = "/var/lib/netbox/static/"; + }; + }; + + users.users.nginx.extraGroups = [ "netbox" ]; + + networking.firewall.allowedTCPPorts = [ 80 ]; + + specialisation.upgrade.configuration.services.netbox.package = lib.mkForce pkgs.netbox; + }; + + testScript = { nodes, ... }: + let + apiVersion = version: lib.pipe version [ + (lib.splitString ".") + (lib.take 2) + (lib.concatStringsSep ".") + ]; + oldApiVersion = apiVersion oldNetbox.version; + newApiVersion = apiVersion pkgs.netbox.version; + in + '' + start_all() + machine.wait_for_unit("netbox.target") + machine.wait_for_unit("nginx.service") + machine.wait_until_succeeds("journalctl --since -1m --unit netbox --grep Listening") + + def api_version(headers): + header = [header for header in headers.splitlines() if header.startswith("API-Version:")][0] + return header.split()[1] + + def check_api_version(version): + headers = machine.succeed( + "curl -sSfL http://localhost/api/ --head -H 'Content-Type: application/json'" + ) + assert api_version(headers) == version + + with subtest("NetBox version is the old one"): + check_api_version("${oldApiVersion}") + + # Somehow, even though netbox-housekeeping.service has After=netbox.service, + # netbox-housekeeping.service and netbox.service still get started at the + # same time, making netbox-housekeeping fail (can't really do some house + # keeping job if the database is not correctly formed). + # + # So we don't check that the upgrade went well, we just check that + # netbox.service is active, and that netbox-housekeeping can be run + # successfully afterwards. + # + # This is not good UX, but the system should be working nonetheless. + machine.execute("${nodes.machine.system.build.toplevel}/specialisation/upgrade/bin/switch-to-configuration test >&2") + + machine.wait_for_unit("netbox.service") + machine.succeed("systemctl start netbox-housekeeping.service") + + with subtest("NetBox version is the new one"): + check_api_version("${newApiVersion}") + ''; +}) From fce8aa18b17ad32edd5b46b99527465a4ff27d5a Mon Sep 17 00:00:00 2001 From: Minijackson Date: Mon, 31 Jul 2023 15:21:07 +0200 Subject: [PATCH 5/6] netbox: 3.5.6 -> 3.5.7 --- pkgs/servers/web-apps/netbox/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/servers/web-apps/netbox/default.nix b/pkgs/servers/web-apps/netbox/default.nix index f68ec501185c..647cf9a34625 100644 --- a/pkgs/servers/web-apps/netbox/default.nix +++ b/pkgs/servers/web-apps/netbox/default.nix @@ -23,8 +23,8 @@ in }; netbox = callPackage generic { - version = "3.5.6"; - hash = "sha256-n5EJQcC5uVoL5KjGzF7bLF8c4Wke/YBJpx2V9KZz5Qo="; + version = "3.5.7"; + hash = "sha256-R5P4FOhn7rE6e9H9U3JlE3ms4Svv6ps4c3+ZjE1KwNM="; extraPatches = [ # Allow setting the STATIC_ROOT from within the configuration and setting a custom redis URL ./config.patch From 1bac8f5aa1046a8882a6dba2ad66f4b11d1ab066 Mon Sep 17 00:00:00 2001 From: Minijackson Date: Mon, 31 Jul 2023 15:27:17 +0200 Subject: [PATCH 6/6] netbox,netbox_3_3: link netbox-upgrade NixOS test --- pkgs/servers/web-apps/netbox/default.nix | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkgs/servers/web-apps/netbox/default.nix b/pkgs/servers/web-apps/netbox/default.nix index 647cf9a34625..e5f99033b5d4 100644 --- a/pkgs/servers/web-apps/netbox/default.nix +++ b/pkgs/servers/web-apps/netbox/default.nix @@ -17,7 +17,10 @@ in }) ]; - tests.netbox = nixosTests.netbox_3_3; + tests = { + netbox = nixosTests.netbox_3_3; + inherit (nixosTests) netbox-upgrade; + }; maintainers = with lib.maintainers; [ n0emis raitobezarius ]; eol = true; }; @@ -30,7 +33,7 @@ in ./config.patch ]; tests = { - inherit (nixosTests) netbox; + inherit (nixosTests) netbox netbox-upgrade; }; maintainers = with lib.maintainers; [ minijackson n0emis raitobezarius ];