nixos/rdma-core: add module for soft RoCE and test (#34607)

nixos/modules/module-list.nix

@@ -531,6 +531,7 @@
   ./services/networking/redsocks.nix
   ./services/networking/resilio.nix
   ./services/networking/rpcbind.nix
+  ./services/networking/rxe.nix
   ./services/networking/sabnzbd.nix
   ./services/networking/searx.nix
   ./services/networking/seeks.nix

nixos/modules/services/networking/rxe.nix

@@ -0,0 +1,63 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.networking.rxe;
+
+  runRxeCmd = cmd: ifcs:
+    concatStrings ( map (x: "${pkgs.rdma-core}/bin/rxe_cfg -n ${cmd} ${x};") ifcs);
+
+  startScript = pkgs.writeShellScriptBin "rxe-start" ''
+    ${pkgs.rdma-core}/bin/rxe_cfg -n start
+    ${runRxeCmd "add" cfg.interfaces}
+    ${pkgs.rdma-core}/bin/rxe_cfg
+  '';
+
+  stopScript = pkgs.writeShellScriptBin "rxe-stop" ''
+    ${runRxeCmd "remove" cfg.interfaces }
+    ${pkgs.rdma-core}/bin/rxe_cfg -n stop
+  '';
+
+in {
+  ###### interface
+
+  options = {
+    networking.rxe = {
+      enable = mkEnableOption "RDMA over converged ethernet";
+      interfaces = mkOption {
+        type = types.listOf types.str;
+        default = [ ];
+        example = [ "eth0" ];
+        description = ''
+          Enable RDMA on the listed interfaces. The corresponding virtual
+          RDMA interfaces will be named rxe0 ... rxeN where the ordering
+          will be as they are named in the list. UDP port 4791 must be
+          open on the respective ethernet interfaces.
+        '';
+      };
+    };
+  };
+
+  ###### implementation
+
+  config = mkIf cfg.enable {
+
+    systemd.services.rxe = {
+      path = with pkgs; [ kmod rdma-core ];
+      description = "RoCE interfaces";
+
+      wantedBy = [ "multi-user.target" ];
+      after = [ "systemd-modules-load.service" "network-online.target" ];
+      wants = [ "network-pre.target" ];
+
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+        ExecStart = "${startScript}/bin/rxe-start";
+        ExecStop = "${stopScript}/bin/rxe-stop";
+      };
+    };
+  };
+}
+

nixos/release.nix

@@ -336,6 +336,7 @@ in rec {
   tests.radicale = callTest tests/radicale.nix {};
   tests.rspamd = callSubTests tests/rspamd.nix {};
   tests.runInMachine = callTest tests/run-in-machine.nix {};
+  tests.rxe = callTest tests/rxe.nix {};
   tests.samba = callTest tests/samba.nix {};
   tests.sddm = callSubTests tests/sddm.nix {};
   tests.simple = callTest tests/simple.nix {};

nixos/tests/rxe.nix

@@ -0,0 +1,53 @@
+import ./make-test.nix ({ pkgs, ... } :
+
+let
+  node = { config, pkgs, lib, ... } : {
+    networking = {
+      firewall = {
+        allowedUDPPorts = [ 4791 ]; # open RoCE port
+        allowedTCPPorts = [ 4800 ]; # port for test utils
+      };
+      rxe = {
+        enable = true;
+        interfaces = [ "eth1" ];
+      };
+    };
+
+    environment.systemPackages = with pkgs; [ rdma-core screen ];
+  };
+
+in {
+  name = "rxe";
+
+  nodes = {
+    server = node;
+    client = node;
+  };
+
+  testScript = ''
+    # Test if rxe interface comes up
+    $server->waitForUnit("default.target");
+    $server->succeed("systemctl status rxe.service");
+    $server->succeed("ibv_devices | grep rxe0");
+
+    $client->waitForUnit("default.target");
+
+    # ping pong test
+    $server->succeed("screen -dmS rc_pingpong ibv_rc_pingpong -p 4800 -g0");
+    $client->succeed("sleep 2; ibv_rc_pingpong -p 4800 -g0 server");
+
+    $server->succeed("screen -dmS uc_pingpong ibv_uc_pingpong -p 4800 -g0");
+    $client->succeed("sleep 2; ibv_uc_pingpong -p 4800 -g0 server");
+
+    $server->succeed("screen -dmS ud_pingpong ibv_ud_pingpong -p 4800 -s 1024 -g0");
+    $client->succeed("sleep 2; ibv_ud_pingpong -p 4800 -s 1024 -g0 server");
+
+    $server->succeed("screen -dmS srq_pingpong ibv_srq_pingpong -p 4800 -g0");
+    $client->succeed("sleep 2; ibv_srq_pingpong -p 4800 -g0 server");
+
+    $server->succeed("screen -dmS rping rping -s -a server -C 10");
+    $client->succeed("sleep 2; rping -c -a server -C 10");
+  '';
+})
+
+