mirror of
https://github.com/NixOS/nixpkgs.git
synced 2024-11-28 08:31:59 +00:00
121cfd1998
The multipath-tools package had existed in Nixpkgs for some time but without a nixos module to configure/drive it. This module provides attributes to drive the majority of multipath configuration options and is being successfully used in stage-1 and stage-2 boot to mount /nix from a multipath-serviced iSCSI volume. Credit goes to @grahamc for early contributions to the module and authoring the NixOS module test.
268 lines
8.9 KiB
Nix
268 lines
8.9 KiB
Nix
import ./make-test-python.nix (
|
|
{ pkgs, lib, ... }:
|
|
let
|
|
initiatorName = "iqn.2020-08.org.linux-iscsi.initiatorhost:example";
|
|
targetName = "iqn.2003-01.org.linux-iscsi.target.x8664:sn.acf8fd9c23af";
|
|
in
|
|
{
|
|
name = "iscsi";
|
|
meta = {
|
|
maintainers = pkgs.lib.teams.deshaw.members;
|
|
};
|
|
|
|
nodes = {
|
|
target = { config, pkgs, lib, ... }: {
|
|
virtualisation.vlans = [ 1 2 ];
|
|
services.target = {
|
|
enable = true;
|
|
config = {
|
|
fabric_modules = [ ];
|
|
storage_objects = [
|
|
{
|
|
dev = "/dev/vdb";
|
|
name = "test";
|
|
plugin = "block";
|
|
write_back = true;
|
|
wwn = "92b17c3f-6b40-4168-b082-ceeb7b495522";
|
|
}
|
|
];
|
|
targets = [
|
|
{
|
|
fabric = "iscsi";
|
|
tpgs = [
|
|
{
|
|
enable = true;
|
|
attributes = {
|
|
authentication = 0;
|
|
generate_node_acls = 1;
|
|
};
|
|
luns = [
|
|
{
|
|
alias = "94dfe06967";
|
|
alua_tg_pt_gp_name = "default_tg_pt_gp";
|
|
index = 0;
|
|
storage_object = "/backstores/block/test";
|
|
}
|
|
];
|
|
node_acls = [
|
|
{
|
|
mapped_luns = [
|
|
{
|
|
alias = "d42f5bdf8a";
|
|
index = 0;
|
|
tpg_lun = 0;
|
|
write_protect = false;
|
|
}
|
|
];
|
|
node_wwn = initiatorName;
|
|
}
|
|
];
|
|
portals = [
|
|
{
|
|
ip_address = "0.0.0.0";
|
|
iser = false;
|
|
offload = false;
|
|
port = 3260;
|
|
}
|
|
];
|
|
tag = 1;
|
|
}
|
|
];
|
|
wwn = targetName;
|
|
}
|
|
];
|
|
};
|
|
};
|
|
|
|
networking.firewall.allowedTCPPorts = [ 3260 ];
|
|
networking.firewall.allowedUDPPorts = [ 3260 ];
|
|
|
|
virtualisation.memorySize = 2048;
|
|
virtualisation.emptyDiskImages = [ 2048 ];
|
|
};
|
|
|
|
initiatorAuto = { nodes, config, pkgs, ... }: {
|
|
virtualisation.vlans = [ 1 2 ];
|
|
|
|
services.multipath = {
|
|
enable = true;
|
|
defaults = ''
|
|
find_multipaths yes
|
|
user_friendly_names yes
|
|
'';
|
|
pathGroups = [
|
|
{
|
|
alias = 123456;
|
|
wwid = "3600140592b17c3f6b404168b082ceeb7";
|
|
}
|
|
];
|
|
};
|
|
|
|
services.openiscsi = {
|
|
enable = true;
|
|
enableAutoLoginOut = true;
|
|
discoverPortal = "target";
|
|
name = initiatorName;
|
|
};
|
|
|
|
environment.systemPackages = with pkgs; [
|
|
xfsprogs
|
|
];
|
|
|
|
environment.etc."initiator-root-disk-closure".source = nodes.initiatorRootDisk.config.system.build.toplevel;
|
|
|
|
nix.binaryCaches = lib.mkForce [ ];
|
|
nix.extraOptions = ''
|
|
hashed-mirrors =
|
|
connect-timeout = 1
|
|
'';
|
|
};
|
|
|
|
initiatorRootDisk = { config, pkgs, modulesPath, lib, ... }: {
|
|
boot.initrd.network.enable = true;
|
|
boot.loader.grub.enable = false;
|
|
|
|
boot.kernelParams = lib.mkOverride 5 (
|
|
[
|
|
"boot.shell_on_fail"
|
|
"console=tty1"
|
|
"ip=192.168.1.1:::255.255.255.0::ens9:none"
|
|
"ip=192.168.2.1:::255.255.255.0::ens10:none"
|
|
]
|
|
);
|
|
|
|
# defaults to true, puts some code in the initrd that tries to mount an overlayfs on /nix/store
|
|
virtualisation.writableStore = false;
|
|
virtualisation.vlans = [ 1 2 ];
|
|
|
|
services.multipath = {
|
|
enable = true;
|
|
defaults = ''
|
|
find_multipaths yes
|
|
user_friendly_names yes
|
|
'';
|
|
pathGroups = [
|
|
{
|
|
alias = 123456;
|
|
wwid = "3600140592b17c3f6b404168b082ceeb7";
|
|
}
|
|
];
|
|
};
|
|
|
|
fileSystems = lib.mkOverride 5 {
|
|
"/" = {
|
|
fsType = "xfs";
|
|
device = "/dev/mapper/123456";
|
|
options = [ "_netdev" ];
|
|
};
|
|
};
|
|
|
|
boot.initrd.extraFiles."etc/multipath/wwids".source = pkgs.writeText "wwids" "/3600140592b17c3f6b404168b082ceeb7/";
|
|
|
|
boot.iscsi-initiator = {
|
|
discoverPortal = "target";
|
|
name = initiatorName;
|
|
target = targetName;
|
|
extraIscsiCommands = ''
|
|
iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login
|
|
'';
|
|
};
|
|
};
|
|
|
|
};
|
|
|
|
testScript = { nodes, ... }: ''
|
|
target.start()
|
|
target.wait_for_unit("iscsi-target.service")
|
|
|
|
initiatorAuto.start()
|
|
|
|
initiatorAuto.wait_for_unit("iscsid.service")
|
|
initiatorAuto.wait_for_unit("iscsi.service")
|
|
initiatorAuto.get_unit_info("iscsi")
|
|
|
|
# Expecting this to fail since we should already know about 192.168.1.3
|
|
initiatorAuto.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
|
|
# Expecting this to succeed since we don't yet know about 192.168.2.3
|
|
initiatorAuto.succeed("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")
|
|
|
|
# /dev/sda is provided by iscsi on target
|
|
initiatorAuto.succeed("set -x; while ! test -e /dev/sda; do sleep 1; done")
|
|
|
|
initiatorAuto.succeed("mkfs.xfs /dev/sda")
|
|
initiatorAuto.succeed("mkdir /mnt")
|
|
|
|
# Start by verifying /dev/sda and /dev/sdb are both the same disk
|
|
initiatorAuto.succeed("mount /dev/sda /mnt")
|
|
initiatorAuto.succeed("touch /mnt/hi")
|
|
initiatorAuto.succeed("umount /mnt")
|
|
|
|
initiatorAuto.succeed("mount /dev/sdb /mnt")
|
|
initiatorAuto.succeed("test -e /mnt/hi")
|
|
initiatorAuto.succeed("umount /mnt")
|
|
|
|
initiatorAuto.succeed("systemctl restart multipathd")
|
|
initiatorAuto.succeed("multipath -ll | systemd-cat")
|
|
|
|
# Install our RootDisk machine to 123456, the alias to the device that multipath is now managing
|
|
initiatorAuto.succeed("mount /dev/mapper/123456 /mnt")
|
|
initiatorAuto.succeed("mkdir -p /mnt/etc/{multipath,iscsi}")
|
|
initiatorAuto.succeed("cp -r /etc/multipath/wwids /mnt/etc/multipath/wwids")
|
|
initiatorAuto.succeed("cp -r /etc/iscsi/{nodes,send_targets} /mnt/etc/iscsi")
|
|
initiatorAuto.succeed(
|
|
"nixos-install --no-bootloader --no-root-passwd --system /etc/initiator-root-disk-closure"
|
|
)
|
|
initiatorAuto.succeed("umount /mnt")
|
|
initiatorAuto.shutdown()
|
|
|
|
initiatorRootDisk.start()
|
|
initiatorRootDisk.wait_for_unit("multi-user.target")
|
|
initiatorRootDisk.wait_for_unit("iscsid")
|
|
|
|
# Log in over both nodes
|
|
initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
|
|
initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")
|
|
initiatorRootDisk.succeed("systemctl restart multipathd")
|
|
initiatorRootDisk.succeed("multipath -ll | systemd-cat")
|
|
|
|
# Verify we can write and sync the root disk
|
|
initiatorRootDisk.succeed("mkdir /scratch")
|
|
initiatorRootDisk.succeed("touch /scratch/both-up")
|
|
initiatorRootDisk.succeed("sync /scratch")
|
|
|
|
# Verify we can write to the root with ens9 (sda, 192.168.1.3) down
|
|
initiatorRootDisk.succeed("ip link set ens9 down")
|
|
initiatorRootDisk.succeed("touch /scratch/ens9-down")
|
|
initiatorRootDisk.succeed("sync /scratch")
|
|
initiatorRootDisk.succeed("ip link set ens9 up")
|
|
|
|
# todo: better way to wait until multipath notices the link is back
|
|
initiatorRootDisk.succeed("sleep 5")
|
|
initiatorRootDisk.succeed("touch /scratch/both-down")
|
|
initiatorRootDisk.succeed("sync /scratch")
|
|
|
|
# Verify we can write to the root with ens10 (sdb, 192.168.2.3) down
|
|
initiatorRootDisk.succeed("ip link set ens10 down")
|
|
initiatorRootDisk.succeed("touch /scratch/ens10-down")
|
|
initiatorRootDisk.succeed("sync /scratch")
|
|
initiatorRootDisk.succeed("ip link set ens10 up")
|
|
initiatorRootDisk.succeed("touch /scratch/ens10-down")
|
|
initiatorRootDisk.succeed("sync /scratch")
|
|
|
|
initiatorRootDisk.succeed("ip link set ens9 up")
|
|
initiatorRootDisk.succeed("ip link set ens10 up")
|
|
initiatorRootDisk.shutdown()
|
|
|
|
# Verify we can boot with the target's eth1 down, forcing
|
|
# it to multipath via the second link
|
|
target.succeed("ip link set eth1 down")
|
|
initiatorRootDisk.start()
|
|
initiatorRootDisk.wait_for_unit("multi-user.target")
|
|
initiatorRootDisk.wait_for_unit("iscsid")
|
|
initiatorRootDisk.succeed("test -e /scratch/both-up")
|
|
'';
|
|
}
|
|
)
|
|
|
|
|