Merge pull request #190181 from RaitoBezarius/garage-module

services/garage: init

nixos/doc/manual/from_md/release-notes/rl-2211.section.xml

@@ -312,6 +312,14 @@
           <link linkend="opt-services.endlessh-go.enable">services.endlessh-go</link>.
         </para>
       </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://garagehq.deuxfleurs.fr/">Garage</link>,
+          a simple object storage server for geodistributed deployments,
+          alternative to MinIO. Available as
+          <link linkend="opt-services.garage.enable">services.garage</link>.
+        </para>
+      </listitem>
       <listitem>
         <para>
           <link xlink:href="https://netbird.io">netbird</link>, a zero

nixos/doc/manual/release-notes/rl-2211.section.md

@@ -108,6 +108,8 @@ In addition to numerous new and upgraded packages, this release has the followin
 
 - [endlessh-go](https://github.com/shizunge/endlessh-go), an SSH tarpit that exposes Prometheus metrics. Available as [services.endlessh-go](#opt-services.endlessh-go.enable).
 
+- [Garage](https://garagehq.deuxfleurs.fr/), a simple object storage server for geodistributed deployments, alternative to MinIO. Available as [services.garage](#opt-services.garage.enable).
+
 - [netbird](https://netbird.io), a zero configuration VPN.
   Available as [services.netbird](options.html#opt-services.netbird.enable).
 

nixos/modules/module-list.nix

@@ -1147,6 +1147,7 @@
   ./services/web-servers/caddy/default.nix
   ./services/web-servers/darkhttpd.nix
   ./services/web-servers/fcgiwrap.nix
+  ./services/web-servers/garage.nix
   ./services/web-servers/hitch/default.nix
   ./services/web-servers/hydron.nix
   ./services/web-servers/jboss/default.nix

nixos/modules/services/web-servers/garage.nix

@@ -0,0 +1,91 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.garage;
+  toml = pkgs.formats.toml {};
+  configFile = toml.generate "garage.toml" cfg.settings;
+in
+{
+  meta.maintainers = [ maintainers.raitobezarius ];
+
+  options.services.garage = {
+    enable = mkEnableOption (lib.mdDoc "Garage Object Storage (S3 compatible)");
+
+    extraEnvironment = mkOption {
+      type = types.attrsOf types.str;
+      description = lib.mdDoc "Extra environment variables to pass to the Garage server.";
+      default = {};
+      example = { RUST_BACKTRACE="yes"; };
+    };
+
+    logLevel = mkOption {
+      type = types.enum (["info" "debug" "trace"]);
+      default = "info";
+      example = "debug";
+      description = lib.mdDoc "Garage log level, see <https://garagehq.deuxfleurs.fr/documentation/quick-start/#launching-the-garage-server> for examples.";
+    };
+
+    settings = mkOption {
+      type = types.submodule {
+        freeformType = toml.type;
+
+        options = {
+          metadata_dir = mkOption {
+            default = "/var/lib/garage/meta";
+            type = types.path;
+            description = lib.mdDoc "The metadata directory, put this on a fast disk (e.g. SSD) if possible.";
+          };
+
+          data_dir = mkOption {
+            default = "/var/lib/garage/data";
+            type = types.path;
+            description = lib.mdDoc "The main data storage, put this on your large storage (e.g. high capacity HDD)";
+          };
+
+          replication_mode = mkOption {
+            default = "none";
+            type = types.enum ([ "none" "1" "2" "3" 1 2 3 ]);
+            apply = v: toString v;
+            description = lib.mdDoc "Garage replication mode, defaults to none, see: <https://garagehq.deuxfleurs.fr/reference_manual/configuration.html#replication_mode> for reference.";
+          };
+        };
+      };
+      description = lib.mdDoc "Garage configuration, see <https://garagehq.deuxfleurs.fr/reference_manual/configuration.html> for reference.";
+    };
+
+    package = mkOption {
+      default = pkgs.garage;
+      defaultText = literalExpression "pkgs.garage";
+      type = types.package;
+      description = lib.mdDoc "Garage package to use.";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    environment.etc."garage.toml" = {
+      source = configFile;
+    };
+
+    environment.systemPackages = [ cfg.package ]; # For administration
+
+    systemd.services.garage = {
+      description = "Garage Object Storage (S3 compatible)";
+      after = [ "network.target" "network-online.target" ];
+      wants = [ "network.target" "network-online.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        ExecStart = "${cfg.package}/bin/garage server";
+
+        StateDirectory = mkIf (hasPrefix "/var/lib/garage" cfg.settings.data_dir && hasPrefix "/var/lib/garage" cfg.settings.metadata_dir) "garage";
+        DynamicUser = lib.mkDefault true;
+        ProtectHome = true;
+        NoNewPrivileges = true;
+      };
+      environment = {
+        RUST_LOG = lib.mkDefault "garage=${cfg.logLevel}";
+      } // cfg.extraEnvironment;
+    };
+  };
+}

nixos/tests/all-tests.nix

@@ -214,6 +214,7 @@ in {
   fsck = handleTest ./fsck.nix {};
   ft2-clone = handleTest ./ft2-clone.nix {};
   mimir = handleTest ./mimir.nix {};
+  garage = handleTest ./garage.nix {};
   gerrit = handleTest ./gerrit.nix {};
   geth = handleTest ./geth.nix {};
   ghostunnel = handleTest ./ghostunnel.nix {};

nixos/tests/garage.nix

@@ -0,0 +1,169 @@
+import ./make-test-python.nix ({ pkgs, ...} :
+let
+    mkNode = { replicationMode, publicV6Address ? "::1" }: { pkgs, ... }: {
+      networking.interfaces.eth1.ipv6.addresses = [{
+        address = publicV6Address;
+        prefixLength = 64;
+      }];
+
+      networking.firewall.allowedTCPPorts = [ 3901 3902 ];
+
+      services.garage = {
+        enable = true;
+        settings = {
+          replication_mode = replicationMode;
+
+          rpc_bind_addr = "[::]:3901";
+          rpc_public_addr = "[${publicV6Address}]:3901";
+          rpc_secret = "5c1915fa04d0b6739675c61bf5907eb0fe3d9c69850c83820f51b4d25d13868c";
+
+          s3_api = {
+            s3_region = "garage";
+            api_bind_addr = "[::]:3900";
+            root_domain = ".s3.garage";
+          };
+
+          s3_web = {
+            bind_addr = "[::]:3902";
+            root_domain = ".web.garage";
+            index = "index.html";
+          };
+        };
+      };
+      environment.systemPackages = [ pkgs.minio-client ];
+
+      # Garage requires at least 1GiB of free disk space to run.
+      virtualisation.diskSize = 2 * 1024;
+    };
+
+
+in {
+  name = "garage";
+  meta = {
+    maintainers = with pkgs.lib.maintainers; [ raitobezarius ];
+  };
+
+  nodes = {
+    single_node = mkNode { replicationMode = "none"; };
+    node1 = mkNode { replicationMode = 3; publicV6Address = "fc00:1::1"; };
+    node2 = mkNode { replicationMode = 3; publicV6Address = "fc00:1::2"; };
+    node3 = mkNode { replicationMode = 3; publicV6Address = "fc00:1::3"; };
+    node4 = mkNode { replicationMode = 3; publicV6Address = "fc00:1::4"; };
+  };
+
+  testScript = ''
+    from typing import List
+    from dataclasses import dataclass
+    import re
+    start_all()
+
+    cur_version_regex = re.compile('Current cluster layout version: (?P<ver>\d*)')
+    key_creation_regex = re.compile('Key name: (?P<key_name>.*)\nKey ID: (?P<key_id>.*)\nSecret key: (?P<secret_key>.*)')
+
+    @dataclass
+    class S3Key:
+       key_name: str
+       key_id: str
+       secret_key: str
+
+    @dataclass
+    class GarageNode:
+       node_id: str
+       host: str
+
+    def get_node_fqn(machine: Machine) -> GarageNode:
+      node_id, host = machine.succeed("garage node id").split('@')
+      return GarageNode(node_id=node_id, host=host)
+
+    def get_node_id(machine: Machine) -> str:
+      return get_node_fqn(machine).node_id
+
+    def get_layout_version(machine: Machine) -> int:
+      version_data = machine.succeed("garage layout show")
+      m = cur_version_regex.search(version_data)
+      if m and m.group('ver') is not None:
+        return int(m.group('ver')) + 1
+      else:
+        raise ValueError('Cannot find current layout version')
+
+    def apply_garage_layout(machine: Machine, layouts: List[str]):
+       for layout in layouts:
+          machine.succeed(f"garage layout assign {layout}")
+       version = get_layout_version(machine)
+       machine.succeed(f"garage layout apply --version {version}")
+
+    def create_api_key(machine: Machine, key_name: str) -> S3Key:
+       output = machine.succeed(f"garage key new --name {key_name}")
+       m = key_creation_regex.match(output)
+       if not m or not m.group('key_id') or not m.group('secret_key'):
+          raise ValueError('Cannot parse API key data')
+       return S3Key(key_name=key_name, key_id=m.group('key_id'), secret_key=m.group('secret_key'))
+
+    def get_api_key(machine: Machine, key_pattern: str) -> S3Key:
+       output = machine.succeed(f"garage key info {key_pattern}")
+       m = key_creation_regex.match(output)
+       if not m or not m.group('key_name') or not m.group('key_id') or not m.group('secret_key'):
+           raise ValueError('Cannot parse API key data')
+       return S3Key(key_name=m.group('key_name'), key_id=m.group('key_id'), secret_key=m.group('secret_key'))
+
+    def test_bucket_writes(node):
+      node.succeed("garage bucket create test-bucket")
+      s3_key = create_api_key(node, "test-api-key")
+      node.succeed("garage bucket allow --read --write test-bucket --key test-api-key")
+      other_s3_key = get_api_key(node, 'test-api-key')
+      assert other_s3_key.secret_key == other_s3_key.secret_key
+      node.succeed(
+        f"mc alias set test-garage http://[::1]:3900 {s3_key.key_id} {s3_key.secret_key} --api S3v4"
+      )
+      node.succeed("echo test | mc pipe test-garage/test-bucket/test.txt")
+      assert node.succeed("mc cat test-garage/test-bucket/test.txt").strip() == "test"
+
+    def test_bucket_over_http(node, bucket='test-bucket', url=None):
+      if url is None:
+         url = f"{bucket}.web.garage"
+
+      node.succeed(f'garage bucket website --allow {bucket}')
+      node.succeed(f'echo hello world | mc pipe test-garage/{bucket}/index.html')
+      assert (node.succeed(f"curl -H 'Host: {url}' http://localhost:3902")).strip() == 'hello world'
+
+    with subtest("Garage works as a single-node S3 storage"):
+      single_node.wait_for_unit("garage.service")
+      single_node.wait_for_open_port(3900)
+      # Now Garage is initialized.
+      single_node_id = get_node_id(single_node)
+      apply_garage_layout(single_node, [f'-z qemutest -c 1 "{single_node_id}"'])
+      # Now Garage is operational.
+      test_bucket_writes(single_node)
+      test_bucket_over_http(single_node)
+
+    with subtest("Garage works as a multi-node S3 storage"):
+      nodes = ('node1', 'node2', 'node3', 'node4')
+      rev_machines = {m.name: m for m in machines}
+      def get_machine(key): return rev_machines[key]
+      for key in nodes:
+        node = get_machine(key)
+        node.wait_for_unit("garage.service")
+        node.wait_for_open_port(3900)
+
+      # Garage is initialized on all nodes.
+      node_ids = {key: get_node_fqn(get_machine(key)) for key in nodes}
+
+      for key in nodes:
+        for other_key in nodes:
+          if other_key != key:
+            other_id = node_ids[other_key]
+            get_machine(key).succeed(f"garage node connect {other_id.node_id}@{other_id.host}")
+
+      # Provide multiple zones for the nodes.
+      zones = ["nixcon", "nixcon", "paris_meetup", "fosdem"]
+      apply_garage_layout(node1,
+      [
+        f'{ndata.node_id} -z {zones[index]} -c 1'
+        for index, ndata in enumerate(node_ids.values())
+      ])
+      # Now Garage is operational.
+      test_bucket_writes(node1)
+      for node in nodes:
+         test_bucket_over_http(get_machine(node))
+  '';
+})