nixpkgs/nixos/tests/prometheus/config-reload.nix

import ../make-test-python.nix (
  { lib, pkgs, ... }:

  {
    name = "prometheus-config-reload";

    nodes = {
      prometheus =
        { config, pkgs, ... }:
        {
          environment.systemPackages = [ pkgs.jq ];

          networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];

          services.prometheus = {
            enable = true;
            enableReload = true;
            globalConfig.scrape_interval = "2s";
            scrapeConfigs = [
              {
                job_name = "prometheus";
                static_configs = [
                  {
                    targets = [
                      "prometheus:${toString config.services.prometheus.port}"
                    ];
                  }
                ];
              }
            ];
          };

          specialisation = {
            "prometheus-config-change" = {
              configuration = {
                environment.systemPackages = [ pkgs.yq ];

                # This configuration just adds a new prometheus job
                # to scrape the node_exporter metrics of the s3 machine.
                services.prometheus = {
                  scrapeConfigs = [
                    {
                      job_name = "node";
                      static_configs = [
                        {
                          targets = [ "node:${toString config.services.prometheus.exporters.node.port}" ];
                        }
                      ];
                    }
                  ];
                };
              };
            };
          };
        };
    };

    testScript = ''
      prometheus.wait_for_unit("prometheus")
      prometheus.wait_for_open_port(9090)

      # Check if switching to a NixOS configuration that changes the prometheus
      # configuration reloads (instead of restarts) prometheus before the switch
      # finishes successfully:
      with subtest("config change reloads prometheus"):
        import json
        # We check if prometheus has finished reloading by looking for the message
        # "Completed loading of configuration file" in the journal between the start
        # and finish of switching to the new NixOS configuration.
        #
        # To mark the start we record the journal cursor before starting the switch:
        cursor_before_switching = json.loads(
            prometheus.succeed("journalctl -n1 -o json --output-fields=__CURSOR")
        )["__CURSOR"]

        # Now we switch:
        prometheus_config_change = prometheus.succeed(
            "readlink /run/current-system/specialisation/prometheus-config-change"
        ).strip()
        prometheus.succeed(prometheus_config_change + "/bin/switch-to-configuration test")

        # Next we retrieve all logs since the start of switching:
        logs_after_starting_switching = prometheus.succeed(
            """
              journalctl --after-cursor='{cursor_before_switching}' -o json --output-fields=MESSAGE
            """.format(
                cursor_before_switching=cursor_before_switching
            )
        )

        # Finally we check if the message "Completed loading of configuration file"
        # occurs before the "finished switching to system configuration" message:
        finished_switching_msg = (
            "finished switching to system configuration " + prometheus_config_change
        )
        reloaded_before_switching_finished = False
        finished_switching = False
        for log_line in logs_after_starting_switching.split("\n"):
            msg = json.loads(log_line)["MESSAGE"]
            if "Completed loading of configuration file" in msg:
                reloaded_before_switching_finished = True
            if msg == finished_switching_msg:
                finished_switching = True
                break

        assert reloaded_before_switching_finished
        assert finished_switching

        # Check if the reloaded config includes the new node job:
        prometheus.succeed(
          """
            curl -sf http://127.0.0.1:9090/api/v1/status/config \
              | jq -r .data.yaml \
              | yq '.scrape_configs | any(.job_name == "node")' \
              | grep true
          """
        )
    '';
  }
)