Merge pull request #101849 from rnhmjoj/searx

nixos/searx: declarative configuration

nixos/doc/manual/release-notes/rl-2103.xml

@@ -402,6 +402,18 @@ http://some.json-exporter.host:7979/probe?target=https://example.com/some/json/e
        SDK licenses if your project requires it. See the androidenv documentation for more details.
      </para>
    </listitem>
+   <listitem>
+     <para>
+      The Searx module has been updated with the ability to configure the
+      service declaratively and uWSGI integration.
+      The option <literal>services.searx.configFile</literal> has been renamed
+      to <xref linkend="opt-services.searx.settingsFile"/> for consistency with
+      the new <xref linkend="opt-services.searx.settings"/>. In addition, the
+      <literal>searx</literal> uid and gid reservations have been removed
+      since they were not necessary: the service is now running with a
+      dynamically allocated uid.
+    </para>
+   </listitem>
   </itemizedlist>
  </section>
 

nixos/modules/misc/ids.nix

@@ -143,7 +143,7 @@ in
       nix-ssh = 104;
       dictd = 105;
       couchdb = 106;
-      searx = 107;
+      #searx = 107; # dynamically allocated as of 2020-10-27
       kippo = 108;
       jenkins = 109;
       systemd-journal-gateway = 110;
@@ -457,7 +457,7 @@ in
       #nix-ssh = 104; # unused
       dictd = 105;
       couchdb = 106;
-      searx = 107;
+      #searx = 107; # dynamically allocated as of 2020-10-27
       kippo = 108;
       jenkins = 109;
       systemd-journal-gateway = 110;

nixos/modules/services/networking/searx.nix

@@ -3,32 +3,133 @@
 with lib;
 
 let
-
+  runDir = "/run/searx";
   cfg = config.services.searx;
 
-  configFile = cfg.configFile;
+  hasEngines =
+    builtins.hasAttr "engines" cfg.settings &&
+    cfg.settings.engines != { };
+
+  # Script to merge NixOS settings with
+  # the default settings.yml bundled in searx.
+  mergeConfig = ''
+    cd ${runDir}
+    # find the default settings.yml
+    default=$(find '${cfg.package}/' -name settings.yml)
+
+    # write NixOS settings as JSON
+    cat <<'EOF' > settings.json
+      ${builtins.toJSON cfg.settings}
+    EOF
+
+    ${optionalString hasEngines ''
+      # extract and convert the default engines array to an object
+      ${pkgs.yq-go}/bin/yq r "$default" engines -j | \
+      ${pkgs.jq}/bin/jq 'reduce .[] as $e ({}; .[$e.name] = $e)' \
+        > engines.json
+
+      # merge and update the NixOS engines with the newly created object
+      cp settings.json temp.json
+      ${pkgs.jq}/bin/jq -s '. as [$s, $e] | $s | .engines |=
+        ($e * . | to_entries | map (.value))' \
+        temp.json engines.json > settings.json
+
+      # clean up temporary files
+      rm {engines,temp}.json
+    ''}
+
+    # merge the default and NixOS settings
+    ${pkgs.yq-go}/bin/yq m -P settings.json "$default" > settings.yml
+    rm settings.json
+
+    # substitute environment variables
+    env -0 | while IFS='=' read -r -d ''' n v; do
+      sed "s#@$n@#$v#g" -i settings.yml
+    done
+
+    # set strict permissions
+    chmod 400 settings.yml
+  '';
 
 in
 
 {
 
+  imports = [
+    (mkRenamedOptionModule
+      [ "services" "searx" "configFile" ]
+      [ "services" "searx" "settingsFile" ])
+  ];
+
   ###### interface
 
   options = {
 
     services.searx = {
 
-      enable = mkEnableOption
-        "the searx server. See https://github.com/asciimoo/searx";
+      enable = mkOption {
+        type = types.bool;
+        default = false;
+        relatedPackages = [ "searx" ];
+        description = "Whether to enable Searx, the meta search engine.";
+      };
 
-      configFile = mkOption {
+      environmentFile = mkOption {
         type = types.nullOr types.path;
         default = null;
-        description = "
-          The path of the Searx server configuration file. If no file
-          is specified, a default file is used (default config file has
-          debug mode enabled).
-        ";
+        description = ''
+          Environment file (see <literal>systemd.exec(5)</literal>
+          "EnvironmentFile=" section for the syntax) to define variables for
+          Searx. This option can be used to safely include secret keys into the
+          Searx configuration.
+        '';
+      };
+
+      settings = mkOption {
+        type = types.attrs;
+        default = { };
+        example = literalExample ''
+          { server.port = 8080;
+            server.bind_address = "0.0.0.0";
+            server.secret_key = "@SEARX_SECRET_KEY@";
+
+            engines.wolframalpha =
+              { shortcut = "wa";
+                api_key = "@WOLFRAM_API_KEY@";
+                engine = "wolframalpha_api";
+              };
+          }
+        '';
+        description = ''
+          Searx settings. These will be merged with (taking precedence over)
+          the default configuration. It's also possible to refer to
+          environment variables
+          (defined in <xref linkend="opt-services.searx.environmentFile"/>)
+          using the syntax <literal>@VARIABLE_NAME@</literal>.
+          <note>
+            <para>
+              For available settings, see the Searx
+              <link xlink:href="https://searx.github.io/searx/admin/settings.html">docs</link>.
+            </para>
+          </note>
+        '';
+      };
+
+      settingsFile = mkOption {
+        type = types.path;
+        default = "${runDir}/settings.yml";
+        description = ''
+          The path of the Searx server settings.yml file. If no file is
+          specified, a default file is used (default config file has debug mode
+          enabled). Note: setting this options overrides
+          <xref linkend="opt-services.searx.settings"/>.
+          <warning>
+            <para>
+              This file, along with any secret key it contains, will be copied
+              into the world-readable Nix store.
+            </para>
+          </warning>
+        '';
       };
 
       package = mkOption {
@@ -38,6 +139,38 @@ in
         description = "searx package to use.";
       };
 
+      runInUwsgi = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Whether to run searx in uWSGI as a "vassal", instead of using its
+          built-in HTTP server. This is the recommended mode for public or
+          large instances, but is unecessary for LAN or local-only use.
+          <warning>
+            <para>
+              The built-in HTTP server logs all queries by default.
+            </para>
+          </warning>
+        '';
+      };
+
+      uwsgiConfig = mkOption {
+        type = types.attrs;
+        default = { http = ":8080"; };
+        example = lib.literalExample ''
+          {
+            disable-logging = true;
+            http = ":8080";                   # serve via HTTP...
+            socket = "/run/searx/searx.sock"; # ...or UNIX socket
+          }
+        '';
+        description = ''
+          Additional configuration of the uWSGI vassal running searx. It
+          should notably specify on which interfaces and ports the vassal
+          should listen.
+        '';
+      };
+
     };
 
   };
@@ -45,33 +178,66 @@ in
 
   ###### implementation
 
-  config = mkIf config.services.searx.enable {
+  config = mkIf cfg.enable {
+    environment.systemPackages = [ cfg.package ];
 
     users.users.searx =
-      { uid = config.ids.uids.searx;
-        description = "Searx user";
-        createHome = true;
-        home = "/var/lib/searx";
+      { description = "Searx daemon user";
+        group = "searx";
+        isSystemUser = true;
       };
 
-    users.groups.searx =
-      { gid = config.ids.gids.searx;
+    users.groups.searx = { };
+
+    systemd.services.searx-init = {
+      description = "Initialise Searx settings";
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+        User = "searx";
+        RuntimeDirectory = "searx";
+        RuntimeDirectoryMode = "750";
+      } // optionalAttrs (cfg.environmentFile != null)
+        { EnvironmentFile = builtins.toPath cfg.environmentFile; };
+      script = mergeConfig;
+    };
+
+    systemd.services.searx = mkIf (!cfg.runInUwsgi) {
+      description = "Searx server, the meta search engine.";
+      wantedBy = [ "network.target" "multi-user.target" ];
+      requires = [ "searx-init.service" ];
+      after = [ "searx-init.service" ];
+      serviceConfig = {
+        User  = "searx";
+        Group = "searx";
+        ExecStart = "${cfg.package}/bin/searx-run";
+      } // optionalAttrs (cfg.environmentFile != null)
+        { EnvironmentFile = builtins.toPath cfg.environmentFile; };
+      environment.SEARX_SETTINGS_PATH = cfg.settingsFile;
+    };
+
+    systemd.services.uwsgi = mkIf (cfg.runInUwsgi)
+      { requires = [ "searx-init.service" ];
+        after = [ "searx-init.service" ];
       };
 
-    systemd.services.searx =
-      {
-        description = "Searx server, the meta search engine.";
-        after = [ "network.target" ];
-        wantedBy = [ "multi-user.target" ];
-        serviceConfig = {
-          User = "searx";
-          ExecStart = "${cfg.package}/bin/searx-run";
-        };
-      } // (optionalAttrs (configFile != null) {
-        environment.SEARX_SETTINGS_PATH = configFile;
-      });
+    services.uwsgi = mkIf (cfg.runInUwsgi) {
+      enable = true;
+      plugins = [ "python3" ];
 
-    environment.systemPackages = [ cfg.package ];
+      instance.type = "emperor";
+      instance.vassals.searx = {
+        type = "normal";
+        strict = true;
+        immediate-uid = "searx";
+        immediate-gid = "searx";
+        lazy-apps = true;
+        enable-threads = true;
+        module = "searx.webapp";
+        env = [ "SEARX_SETTINGS_PATH=${cfg.settingsFile}" ];
+        pythonPackages = self: [ cfg.package ];
+      } // cfg.uwsgiConfig;
+    };
 
   };
 

nixos/tests/all-tests.nix

@@ -342,6 +342,7 @@ in
   sbt-extras = handleTest ./sbt-extras.nix {};
   scala = handleTest ./scala.nix {};
   sddm = handleTest ./sddm.nix {};
+  searx = handleTest ./searx.nix {};
   service-runner = handleTest ./service-runner.nix {};
   shadow = handleTest ./shadow.nix {};
   shadowsocks = handleTest ./shadowsocks {};

nixos/tests/searx.nix

@@ -0,0 +1,109 @@
+import ./make-test-python.nix ({ pkgs, ...} :
+
+{
+  name = "searx";
+  meta = with pkgs.stdenv.lib.maintainers; {
+    maintainers = [ rnhmjoj ];
+  };
+
+  # basic setup: searx running the built-in webserver
+  nodes.base = { ... }: {
+    imports = [ ../modules/profiles/minimal.nix ];
+
+    services.searx = {
+      enable = true;
+      environmentFile = pkgs.writeText "secrets" ''
+        WOLFRAM_API_KEY  = sometoken
+        SEARX_SECRET_KEY = somesecret
+      '';
+
+      settings.server =
+        { port = "8080";
+          bind_address = "0.0.0.0";
+          secret_key = "@SEARX_SECRET_KEY@";
+        };
+      settings.engines = {
+        wolframalpha =
+          { api_key = "@WOLFRAM_API_KEY@";
+            engine = "wolframalpha_api";
+          };
+        startpage.shortcut = "start";
+      };
+    };
+
+  };
+
+  # fancy setup: run in uWSGI and use nginx as proxy
+  nodes.fancy = { ... }: {
+    imports = [ ../modules/profiles/minimal.nix ];
+
+    services.searx = {
+      enable = true;
+      runInUwsgi = true;
+      uwsgiConfig = {
+        # serve using the uwsgi protocol
+        socket = "/run/searx/uwsgi.sock";
+        chmod-socket = "660";
+
+        # use /searx as url "mountpoint"
+        mount = "/searx=searx.webapp:application";
+        module = "";
+        manage-script-name = true;
+      };
+    };
+
+    # use nginx as reverse proxy
+    services.nginx.enable = true;
+    services.nginx.virtualHosts.localhost = {
+      locations."/searx".extraConfig =
+        ''
+          include ${pkgs.nginx}/conf/uwsgi_params;
+          uwsgi_pass unix:/run/searx/uwsgi.sock;
+        '';
+      locations."/searx/static/".alias = "${pkgs.searx}/share/static/";
+    };
+
+    # allow nginx access to the searx socket
+    users.users.nginx.extraGroups = [ "searx" ];
+
+  };
+
+  testScript =
+    ''
+      base.start()
+
+      with subtest("Settings have been merged"):
+          base.wait_for_unit("searx-init")
+          base.wait_for_file("/run/searx/settings.yml")
+          output = base.succeed(
+              "${pkgs.yq-go}/bin/yq r /run/searx/settings.yml"
+              " 'engines.(name==startpage).shortcut'"
+          ).strip()
+          assert output == "start", "Settings not merged"
+
+      with subtest("Environment variables have been substituted"):
+          base.succeed("grep -q somesecret /run/searx/settings.yml")
+          base.succeed("grep -q sometoken /run/searx/settings.yml")
+          base.copy_from_vm("/run/searx/settings.yml")
+
+      with subtest("Basic setup is working"):
+          base.wait_for_open_port(8080)
+          base.wait_for_unit("searx")
+          base.succeed(
+              "${pkgs.curl}/bin/curl --fail http://localhost:8080"
+          )
+          base.shutdown()
+
+      with subtest("Nginx+uWSGI setup is working"):
+          fancy.start()
+          fancy.wait_for_open_port(80)
+          fancy.wait_for_unit("uwsgi")
+          fancy.succeed(
+              "${pkgs.curl}/bin/curl --fail http://localhost/searx >&2"
+          )
+          fancy.succeed(
+              "${pkgs.curl}/bin/curl --fail http://localhost/searx/static/js/bootstrap.min.js >&2"
+          )
+    '';
+})
+

pkgs/servers/web-apps/searx/default.nix

@@ -1,8 +1,8 @@
-{ lib, python3Packages, fetchFromGitHub, fetchpatch }:
+{ lib, nixosTests, python3, python3Packages, fetchFromGitHub, fetchpatch }:
 
 with python3Packages;
 
-buildPythonApplication rec {
+toPythonModule (buildPythonApplication rec {
   pname = "searx";
   version = "0.17.0";
 
@@ -34,10 +34,18 @@ buildPythonApplication rec {
     rm tests/test_robot.py # A variable that is imported is commented out
   '';
 
+  postInstall = ''
+    # Create a symlink for easier access to static data
+    mkdir -p $out/share
+    ln -s ../${python3.sitePackages}/searx/static $out/share/
+  '';
+
+  passthru.tests = { inherit (nixosTests) searx; };
+
   meta = with lib; {
     homepage = "https://github.com/asciimoo/searx";
     description = "A privacy-respecting, hackable metasearch engine";
     license = licenses.agpl3Plus;
     maintainers = with maintainers; [ matejc fpletz globin danielfullmer ];
   };
-}
+})