From 363b0fd040ed58cbd394cebcfb6d2a77b9672d5f Mon Sep 17 00:00:00 2001 From: Jan Malakhovski Date: Wed, 25 Nov 2015 19:07:19 +0000 Subject: [PATCH 1/6] lib: introduce listDfs and toposort, add example to hasPrefix --- lib/lists.nix | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/lib/lists.nix b/lib/lists.nix index 78ffa753ac33..4bf732b88c9a 100644 --- a/lib/lists.nix +++ b/lib/lists.nix @@ -256,6 +256,86 @@ rec { reverseList = xs: let l = length xs; in genList (n: elemAt xs (l - n - 1)) l; + /* Depth-First Search (DFS) for lists `list != []`. + + `before a b == true` means that `b` depends on `a` (there's an + edge from `b` to `a`). + + Examples: + + listDfs true hasPrefix [ "/home/user" "other" "/" "/home" ] + == { minimal = "/"; # minimal element + visited = [ "/home/user" ]; # seen elements (in reverse order) + rest = [ "/home" "other" ]; # everything else + } + + listDfs true hasPrefix [ "/home/user" "other" "/" "/home" "/" ] + == { cycle = "/"; # cycle encountered at this element + loops = [ "/" ]; # and continues to these elements + visited = [ "/" "/home/user" ]; # elements leading to the cycle (in reverse order) + rest = [ "/home" "other" ]; # everything else + + */ + + listDfs = stopOnCycles: before: list: + let + dfs' = us: visited: rest: + let + c = filter (x: before x us) visited; + b = partition (x: before x us) rest; + in if stopOnCycles && (length c > 0) + then { cycle = us; loops = c; inherit visited rest; } + else if length b.right == 0 + then # nothing is before us + { minimal = us; inherit visited rest; } + else # grab the first one before us and continue + dfs' (head b.right) + ([ us ] ++ visited) + (tail b.right ++ b.wrong); + in dfs' (head list) [] (tail list); + + /* Sort a list based on a partial ordering using DFS. This + implementation is O(N^2), if your ordering is linear, use `sort` + instead. + + `before a b == true` means that `b` should be after `a` + in the result. + + Examples: + + toposort hasPrefix [ "/home/user" "other" "/" "/home" ] + == { result = [ "/" "/home" "/home/user" "other" ]; } + + toposort hasPrefix [ "/home/user" "other" "/" "/home" "/" ] + == { cycle = [ "/home/user" "/" "/" ]; # path leading to a cycle + loops = [ "/" ]; } # loops back to these elements + + toposort hasPrefix [ "other" "/home/user" "/home" "/" ] + == { result = [ "other" "/" "/home" "/home/user" ]; } + + toposort (a: b: a < b) [ 3 2 1 ] == { result = [ 1 2 3 ]; } + + */ + + toposort = before: list: + let + dfsthis = listDfs true before list; + toporest = toposort before (dfsthis.visited ++ dfsthis.rest); + in + if length list < 2 + then # finish + { result = list; } + else if dfsthis ? "cycle" + then # there's a cycle, starting from the current vertex, return it + { cycle = reverseList ([ dfsthis.cycle ] ++ dfsthis.visited); + inherit (dfsthis) loops; } + else if toporest ? "cycle" + then # there's a cycle somewhere else in the graph, return it + toporest + # Slow, but short. Can be made a bit faster with an explicit stack. + else # there are no cycles + { result = [ dfsthis.minimal ] ++ toporest.result; }; + /* Sort a list based on a comparator function which compares two elements and returns true if the first argument is strictly below the second argument. The returned list is sorted in an increasing From 1266852fd8d8143bbe38caff65a3ea7577ffe04a Mon Sep 17 00:00:00 2001 From: Jan Malakhovski Date: Tue, 23 Aug 2016 17:41:31 +0000 Subject: [PATCH 2/6] Revert a soon to be useless pice of "nixos/stage-1: add mechanism which lustrates all impurities from / (#17784)" This reverts a pice of commit 3d16af70bf894ce15ec9bdcad3c9ac736dc43630. --- nixos/modules/system/boot/stage-1.nix | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/nixos/modules/system/boot/stage-1.nix b/nixos/modules/system/boot/stage-1.nix index 9be7ad4ae077..baeba1d6b31d 100644 --- a/nixos/modules/system/boot/stage-1.nix +++ b/nixos/modules/system/boot/stage-1.nix @@ -131,16 +131,9 @@ let # The initrd only has to mount / or any FS marked as necessary for # booting (such as the FS containing /nix/store, or an FS needed for # mounting /, like / on a loopback). - # - # We need to guarantee that / is the first filesystem in the list so - # that if and when lustrateRoot is invoked, nothing else is mounted - fileSystems = let - filterNeeded = filter - (fs: fs.mountPoint != "/" && (fs.neededForBoot || elem fs.mountPoint [ "/nix" "/nix/store" "/var" "/var/log" "/var/lib" "/etc" ])); - filterRoot = filter - (fs: fs.mountPoint == "/"); - allFileSystems = attrValues config.fileSystems; - in (filterRoot allFileSystems) ++ (filterNeeded allFileSystems); + fileSystems = filter + (fs: fs.neededForBoot || elem fs.mountPoint [ "/" "/nix" "/nix/store" "/var" "/var/log" "/var/lib" "/etc" ]) + (attrValues config.fileSystems); udevRules = pkgs.stdenv.mkDerivation { From 2c8ca0d1bd1df838b3e3b2d2e17c6855b3b520c1 Mon Sep 17 00:00:00 2001 From: Jan Malakhovski Date: Wed, 25 Nov 2015 19:09:43 +0000 Subject: [PATCH 3/6] nixos: tasks/fileSystems: cleanup --- nixos/modules/tasks/filesystems.nix | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/nixos/modules/tasks/filesystems.nix b/nixos/modules/tasks/filesystems.nix index cf8232c36154..b0abf5eda871 100644 --- a/nixos/modules/tasks/filesystems.nix +++ b/nixos/modules/tasks/filesystems.nix @@ -208,14 +208,15 @@ in formatDevice = fs: let - mountPoint' = escapeSystemdPath fs.mountPoint; - device' = escapeSystemdPath fs.device; + mountPoint' = "${escapeSystemdPath fs.mountPoint}.mount"; + device' = escapeSystemdPath fs.device; + device'' = "${device}.device"; in nameValuePair "mkfs-${device'}" { description = "Initialisation of Filesystem ${fs.device}"; - wantedBy = [ "${mountPoint'}.mount" ]; - before = [ "${mountPoint'}.mount" "systemd-fsck@${device'}.service" ]; - requires = [ "${device'}.device" ]; - after = [ "${device'}.device" ]; + wantedBy = [ mountPoint' ]; + before = [ mountPoint' "systemd-fsck@${device'}.service" ]; + requires = [ device'' ]; + after = [ device'' ]; path = [ pkgs.utillinux ] ++ config.system.fsPackages; script = '' From 65d26c4dc12f8f0113b6b128573f18492ac5b6f6 Mon Sep 17 00:00:00 2001 From: Jan Malakhovski Date: Wed, 25 Nov 2015 19:09:09 +0000 Subject: [PATCH 4/6] nixos: apply toposort to fileSystems to support bind and move mounts And use new `config.system.build.fileSystems` property everywhere. --- nixos/lib/utils.nix | 5 +++++ nixos/modules/security/grsecurity.nix | 2 +- nixos/modules/system/boot/stage-1.nix | 23 ++++++++++---------- nixos/modules/tasks/encrypted-devices.nix | 2 +- nixos/modules/tasks/filesystems.nix | 26 ++++++++++++++++++++--- nixos/modules/tasks/filesystems/zfs.nix | 4 ++-- 6 files changed, 43 insertions(+), 19 deletions(-) diff --git a/nixos/lib/utils.nix b/nixos/lib/utils.nix index 40d0854d968d..56a1e8a1d8b9 100644 --- a/nixos/lib/utils.nix +++ b/nixos/lib/utils.nix @@ -2,6 +2,11 @@ pkgs: with pkgs.lib; rec { + # Check whenever `b` depends on `a` as a fileSystem + # FIXME: it's incorrect to simply use hasPrefix here: "/dev/a" is not a parent of "/dev/ab" + fsBefore = a: b: ((any (x: elem x [ "bind" "move" ]) b.options) && (a.mountPoint == b.device)) + || (hasPrefix a.mountPoint b.mountPoint); + # Escape a path according to the systemd rules, e.g. /dev/xyzzy # becomes dev-xyzzy. FIXME: slow. escapeSystemdPath = s: diff --git a/nixos/modules/security/grsecurity.nix b/nixos/modules/security/grsecurity.nix index 9a2f62a14889..c6332ca9f9f6 100644 --- a/nixos/modules/security/grsecurity.nix +++ b/nixos/modules/security/grsecurity.nix @@ -12,7 +12,7 @@ let (fs: (fs.neededForBoot || elem fs.mountPoint [ "/" "/nix" "/nix/store" "/var" "/var/log" "/var/lib" "/etc" ]) && fs.fsType == "zfs") - (attrValues config.fileSystems) != []; + config.system.build.fileSystems != []; # Ascertain whether NixOS container support is required containerSupportRequired = diff --git a/nixos/modules/system/boot/stage-1.nix b/nixos/modules/system/boot/stage-1.nix index baeba1d6b31d..f26412103ed6 100644 --- a/nixos/modules/system/boot/stage-1.nix +++ b/nixos/modules/system/boot/stage-1.nix @@ -3,7 +3,7 @@ # the modules necessary to mount the root file system, then calls the # init in the root file system to start the second boot stage. -{ config, lib, pkgs, ... }: +{ config, lib, utils, pkgs, ... }: with lib; @@ -23,6 +23,14 @@ let }; + # The initrd only has to mount `/` or any FS marked as necessary for + # booting (such as the FS containing `/nix/store`, or an FS needed for + # mounting `/`, like `/` on a loopback). + fileSystems = filter + (fs: fs.neededForBoot || elem fs.mountPoint [ "/" "/nix" "/nix/store" "/var" "/var/log" "/var/lib" "/etc" ]) + config.system.build.fileSystems; + + # Some additional utilities needed in stage 1, like mount, lvm, fsck # etc. We don't want to bring in all of those packages, so we just # copy what we need. Instead of using statically linked binaries, @@ -71,7 +79,7 @@ let ln -sf kmod $out/bin/modprobe # Copy resize2fs if needed. - ${optionalString (any (fs: fs.autoResize) (attrValues config.fileSystems)) '' + ${optionalString (any (fs: fs.autoResize) config.system.build.fileSystems) '' # We need mke2fs in the initrd. copy_bin_and_libs ${pkgs.e2fsprogs}/sbin/resize2fs ''} @@ -128,14 +136,6 @@ let ''; # */ - # The initrd only has to mount / or any FS marked as necessary for - # booting (such as the FS containing /nix/store, or an FS needed for - # mounting /, like / on a loopback). - fileSystems = filter - (fs: fs.neededForBoot || elem fs.mountPoint [ "/" "/nix" "/nix/store" "/var" "/var/log" "/var/lib" "/etc" ]) - (attrValues config.fileSystems); - - udevRules = pkgs.stdenv.mkDerivation { name = "udev-rules"; allowedReferences = [ extraUtils ]; @@ -398,9 +398,8 @@ in }; config = mkIf (!config.boot.isContainer) { - assertions = [ - { assertion = any (fs: fs.mountPoint == "/") (attrValues config.fileSystems); + { assertion = any (fs: fs.mountPoint == "/") fileSystems; message = "The ‘fileSystems’ option does not specify your root file system."; } { assertion = let inherit (config.boot) resumeDevice; in diff --git a/nixos/modules/tasks/encrypted-devices.nix b/nixos/modules/tasks/encrypted-devices.nix index 457b86e95ab5..b1a7711ddcb4 100644 --- a/nixos/modules/tasks/encrypted-devices.nix +++ b/nixos/modules/tasks/encrypted-devices.nix @@ -3,7 +3,7 @@ with lib; let - fileSystems = attrValues config.fileSystems ++ config.swapDevices; + fileSystems = config.system.build.fileSystems ++ config.swapDevices; encDevs = filter (dev: dev.encrypted.enable) fileSystems; keyedEncDevs = filter (dev: dev.encrypted.keyFile != null) encDevs; keylessEncDevs = filter (dev: dev.encrypted.keyFile == null) encDevs; diff --git a/nixos/modules/tasks/filesystems.nix b/nixos/modules/tasks/filesystems.nix index b0abf5eda871..78dca662dc9d 100644 --- a/nixos/modules/tasks/filesystems.nix +++ b/nixos/modules/tasks/filesystems.nix @@ -5,7 +5,16 @@ with utils; let - fileSystems = attrValues config.fileSystems; + fileSystems' = toposort fsBefore (attrValues config.fileSystems); + + fileSystems = if fileSystems' ? "result" + then # use topologically sorted fileSystems everywhere + fileSystems'.result + else # the assertion below will catch this, + # but we fall back to the original order + # anyway so that other modules could check + # their assertions too + (attrValues config.fileSystems); prioOption = prio: optionalString (prio != null) " pri=${toString prio}"; @@ -162,6 +171,17 @@ in config = { + assertions = let + ls = sep: concatMapStringsSep sep (x: x.mountPoint); + in [ + { assertion = ! (fileSystems' ? "cycle"); + message = "The ‘fileSystems’ option can't be topologically sorted: mountpoint dependency path ${ls " -> " fileSystems'.cycle} loops to ${ls ", " fileSystems'.loops}"; + } + ]; + + # Export for use in other modules + system.build.fileSystems = fileSystems; + boot.supportedFilesystems = map (fs: fs.fsType) fileSystems; # Add the mount helpers to the system path so that `mount' can find them. @@ -177,7 +197,7 @@ in # This is a generated file. Do not edit! # Filesystems. - ${flip concatMapStrings fileSystems (fs: + ${concatMapStrings (fs: (if fs.device != null then fs.device else if fs.label != null then "/dev/disk/by-label/${fs.label}" else throw "No device specified for mount point ‘${fs.mountPoint}’.") @@ -188,7 +208,7 @@ in + " " + (if skipCheck fs then "0" else if fs.mountPoint == "/" then "1" else "2") + "\n" - )} + ) fileSystems} # Swap devices. ${flip concatMapStrings config.swapDevices (sw: diff --git a/nixos/modules/tasks/filesystems/zfs.nix b/nixos/modules/tasks/filesystems/zfs.nix index 4ff3ffc74b16..77059fa43ffa 100644 --- a/nixos/modules/tasks/filesystems/zfs.nix +++ b/nixos/modules/tasks/filesystems/zfs.nix @@ -36,7 +36,7 @@ let fsToPool = fs: datasetToPool fs.device; - zfsFilesystems = filter (x: x.fsType == "zfs") (attrValues config.fileSystems); + zfsFilesystems = filter (x: x.fsType == "zfs") config.system.build.fileSystems; isRoot = fs: fs.neededForBoot || elem fs.mountPoint [ "/" "/nix" "/nix/store" "/var" "/var/log" "/var/lib" "/etc" ]; @@ -277,7 +277,7 @@ in systemd.services = let getPoolFilesystems = pool: - filter (x: x.fsType == "zfs" && (fsToPool x) == pool) (attrValues config.fileSystems); + filter (x: x.fsType == "zfs" && (fsToPool x) == pool) config.system.build.fileSystems; getPoolMounts = pool: let From 8da59c406cea9ee3fbe37fce6211421281c47b78 Mon Sep 17 00:00:00 2001 From: Jan Malakhovski Date: Tue, 23 Aug 2016 17:26:13 +0000 Subject: [PATCH 5/6] nixos: copy resize2fs only for stage-1 fileSystems --- nixos/modules/system/boot/stage-1.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/system/boot/stage-1.nix b/nixos/modules/system/boot/stage-1.nix index f26412103ed6..982f4e4a85c9 100644 --- a/nixos/modules/system/boot/stage-1.nix +++ b/nixos/modules/system/boot/stage-1.nix @@ -79,7 +79,7 @@ let ln -sf kmod $out/bin/modprobe # Copy resize2fs if needed. - ${optionalString (any (fs: fs.autoResize) config.system.build.fileSystems) '' + ${optionalString (any (fs: fs.autoResize) fileSystems) '' # We need mke2fs in the initrd. copy_bin_and_libs ${pkgs.e2fsprogs}/sbin/resize2fs ''} From b267785c43547dbe854c994a91b9f012c9b7812f Mon Sep 17 00:00:00 2001 From: Jan Malakhovski Date: Tue, 23 Aug 2016 18:01:35 +0000 Subject: [PATCH 6/6] nixos: generalize copy-paste from stage-1 and zfs to utils --- nixos/lib/utils.nix | 4 ++++ nixos/modules/system/boot/stage-1.nix | 4 +--- nixos/modules/tasks/filesystems/zfs.nix | 4 +--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nixos/lib/utils.nix b/nixos/lib/utils.nix index 56a1e8a1d8b9..1ef915d40612 100644 --- a/nixos/lib/utils.nix +++ b/nixos/lib/utils.nix @@ -2,6 +2,10 @@ pkgs: with pkgs.lib; rec { + # Check whenever fileSystem is needed for boot + fsNeededForBoot = fs: fs.neededForBoot + || elem fs.mountPoint [ "/" "/nix" "/nix/store" "/var" "/var/log" "/var/lib" "/etc" ]; + # Check whenever `b` depends on `a` as a fileSystem # FIXME: it's incorrect to simply use hasPrefix here: "/dev/a" is not a parent of "/dev/ab" fsBefore = a: b: ((any (x: elem x [ "bind" "move" ]) b.options) && (a.mountPoint == b.device)) diff --git a/nixos/modules/system/boot/stage-1.nix b/nixos/modules/system/boot/stage-1.nix index 982f4e4a85c9..a5c05f3dbbaf 100644 --- a/nixos/modules/system/boot/stage-1.nix +++ b/nixos/modules/system/boot/stage-1.nix @@ -26,9 +26,7 @@ let # The initrd only has to mount `/` or any FS marked as necessary for # booting (such as the FS containing `/nix/store`, or an FS needed for # mounting `/`, like `/` on a loopback). - fileSystems = filter - (fs: fs.neededForBoot || elem fs.mountPoint [ "/" "/nix" "/nix/store" "/var" "/var/log" "/var/lib" "/etc" ]) - config.system.build.fileSystems; + fileSystems = filter utils.fsNeededForBoot config.system.build.fileSystems; # Some additional utilities needed in stage 1, like mount, lvm, fsck diff --git a/nixos/modules/tasks/filesystems/zfs.nix b/nixos/modules/tasks/filesystems/zfs.nix index 77059fa43ffa..c5f41cc338cf 100644 --- a/nixos/modules/tasks/filesystems/zfs.nix +++ b/nixos/modules/tasks/filesystems/zfs.nix @@ -38,11 +38,9 @@ let zfsFilesystems = filter (x: x.fsType == "zfs") config.system.build.fileSystems; - isRoot = fs: fs.neededForBoot || elem fs.mountPoint [ "/" "/nix" "/nix/store" "/var" "/var/log" "/var/lib" "/etc" ]; - allPools = unique ((map fsToPool zfsFilesystems) ++ cfgZfs.extraPools); - rootPools = unique (map fsToPool (filter isRoot zfsFilesystems)); + rootPools = unique (map fsToPool (filter fsNeededForBoot zfsFilesystems)); dataPools = unique (filter (pool: !(elem pool rootPools)) allPools);