nixpkgs/lib/fileset/internal.nix

{ lib ? import ../. }:
let

  inherit (builtins)
    isAttrs
    isPath
    isString
    pathExists
    readDir
    seq
    split
    trace
    typeOf
    ;

  inherit (lib.attrsets)
    attrNames
    attrValues
    mapAttrs
    setAttrByPath
    zipAttrsWith
    ;

  inherit (lib.filesystem)
    pathType
    ;

  inherit (lib.lists)
    all
    commonPrefix
    drop
    elemAt
    filter
    findFirst
    findFirstIndex
    foldl'
    head
    length
    sublist
    tail
    ;

  inherit (lib.path)
    append
    splitRoot
    ;

  inherit (lib.path.subpath)
    components
    join
    ;

  inherit (lib.strings)
    isStringLike
    concatStringsSep
    substring
    stringLength
    ;

in
# Rare case of justified usage of rec:
# - This file is internal, so the return value doesn't matter, no need to make things overridable
# - The functions depend on each other
# - We want to expose all of these functions for easy testing
rec {

  # If you change the internal representation, make sure to:
  # - Increment this version
  # - Add an additional migration function below
  # - Update the description of the internal representation in ./README.md
  _currentVersion = 3;

  # Migrations between versions. The 0th element converts from v0 to v1, and so on
  migrations = [
    # Convert v0 into v1: Add the _internalBase{Root,Components} attributes
    (
      filesetV0:
      let
        parts = splitRoot filesetV0._internalBase;
      in
      filesetV0 // {
        _internalVersion = 1;
        _internalBaseRoot = parts.root;
        _internalBaseComponents = components parts.subpath;
      }
    )

    # Convert v1 into v2: filesetTree's can now also omit attributes to signal paths not being included
    (
      filesetV1:
      # This change is backwards compatible (but not forwards compatible, so we still need a new version)
      filesetV1 // {
        _internalVersion = 2;
      }
    )

    # Convert v2 into v3: filesetTree's now have a representation for an empty file set without a base path
    (
      filesetV2:
      filesetV2 // {
        # All v1 file sets are not the new empty file set
        _internalIsEmptyWithoutBase = false;
        _internalVersion = 3;
      }
    )
  ];

  _noEvalMessage = ''
    lib.fileset: Directly evaluating a file set is not supported.
      To turn it into a usable source, use `lib.fileset.toSource`.
      To pretty-print the contents, use `lib.fileset.trace` or `lib.fileset.traceVal`.'';

  # The empty file set without a base path
  _emptyWithoutBase = {
    _type = "fileset";

    _internalVersion = _currentVersion;

    # The one and only!
    _internalIsEmptyWithoutBase = true;

    # Due to alphabetical ordering, this is evaluated last,
    # which makes the nix repl output nicer than if it would be ordered first.
    # It also allows evaluating it strictly up to this error, which could be useful
    _noEval = throw _noEvalMessage;
  };

  # Create a fileset, see ./README.md#fileset
  # Type: path -> filesetTree -> fileset
  _create = base: tree:
    let
      # Decompose the base into its components
      # See ../path/README.md for why we're not just using `toString`
      parts = splitRoot base;
    in
    {
      _type = "fileset";

      _internalVersion = _currentVersion;

      _internalIsEmptyWithoutBase = false;
      _internalBase = base;
      _internalBaseRoot = parts.root;
      _internalBaseComponents = components parts.subpath;
      _internalTree = tree;

      # Due to alphabetical ordering, this is evaluated last,
      # which makes the nix repl output nicer than if it would be ordered first.
      # It also allows evaluating it strictly up to this error, which could be useful
      _noEval = throw _noEvalMessage;
    };

  # Coerce a value to a fileset, erroring when the value cannot be coerced.
  # The string gives the context for error messages.
  # Type: String -> (fileset | Path) -> fileset
  _coerce = context: value:
    if value._type or "" == "fileset" then
      if value._internalVersion > _currentVersion then
        throw ''
          ${context} is a file set created from a future version of the file set library with a different internal representation:
              - Internal version of the file set: ${toString value._internalVersion}
              - Internal version of the library: ${toString _currentVersion}
              Make sure to update your Nixpkgs to have a newer version of `lib.fileset`.''
      else if value._internalVersion < _currentVersion then
        let
          # Get all the migration functions necessary to convert from the old to the current version
          migrationsToApply = sublist value._internalVersion (_currentVersion - value._internalVersion) migrations;
        in
        foldl' (value: migration: migration value) value migrationsToApply
      else
        value
    else if ! isPath value then
      if isStringLike value then
        throw ''
          ${context} ("${toString value}") is a string-like value, but it should be a file set or a path instead.
              Paths represented as strings are not supported by `lib.fileset`, use `lib.sources` or derivations instead.''
      else
        throw ''
          ${context} is of type ${typeOf value}, but it should be a file set or a path instead.''
    else if ! pathExists value then
      throw ''
        ${context} (${toString value}) does not exist.''
    else
      _singleton value;

  # Coerce many values to filesets, erroring when any value cannot be coerced,
  # or if the filesystem root of the values doesn't match.
  # Type: String -> [ { context :: String, value :: fileset | Path } ] -> [ fileset ]
  _coerceMany = functionContext: list:
    let
      filesets = map ({ context, value }:
        _coerce "${functionContext}: ${context}" value
      ) list;

      # Find the first value with a base, there may be none!
      firstWithBase = findFirst (fileset: ! fileset._internalIsEmptyWithoutBase) null filesets;
      # This value is only accessed if first != null
      firstBaseRoot = firstWithBase._internalBaseRoot;

      # Finds the first element with a filesystem root different than the first element, if any
      differentIndex = findFirstIndex (fileset:
        # The empty value without a base doesn't have a base path
        ! fileset._internalIsEmptyWithoutBase
        && firstBaseRoot != fileset._internalBaseRoot
      ) null filesets;
    in
    # Only evaluates `differentIndex` if there are any elements with a base
    if firstWithBase != null && differentIndex != null then
      throw ''
        ${functionContext}: Filesystem roots are not the same:
            ${(head list).context}: root "${toString firstBaseRoot}"
            ${(elemAt list differentIndex).context}: root "${toString (elemAt filesets differentIndex)._internalBaseRoot}"
            Different roots are not supported.''
    else
      filesets;

  # Create a file set from a path.
  # Type: Path -> fileset
  _singleton = path:
    let
      type = pathType path;
    in
    if type == "directory" then
      _create path type
    else
      # This turns a file path ./default.nix into a fileset with
      # - _internalBase: ./.
      # - _internalTree: {
      #     "default.nix" = <type>;
      #   }
      # See ./README.md#single-files
      _create (dirOf path)
        {
          ${baseNameOf path} = type;
        };

  # Expand a directory representation to an equivalent one in attribute set form.
  # All directory entries are included in the result.
  # Type: Path -> filesetTree -> { <name> = filesetTree; }
  _directoryEntries = path: value:
    if value == "directory" then
      readDir path
    else
      # Set all entries not present to null
      mapAttrs (name: value: null) (readDir path)
      // value;

  /*
    A normalisation of a filesetTree suitable filtering with `builtins.path`:
    - Replace all directories that have no files with `null`.
      This removes directories that would be empty
    - Replace all directories with all files with `"directory"`.
      This speeds up the source filter function

    Note that this function is strict, it evaluates the entire tree

    Type: Path -> filesetTree -> filesetTree
  */
  _normaliseTreeFilter = path: tree:
    if tree == "directory" || isAttrs tree then
      let
        entries = _directoryEntries path tree;
        normalisedSubtrees = mapAttrs (name: _normaliseTreeFilter (path + "/${name}")) entries;
        subtreeValues = attrValues normalisedSubtrees;
      in
      # This triggers either when all files in a directory are filtered out
      # Or when the directory doesn't contain any files at all
      if all isNull subtreeValues then
        null
      # Triggers when we have the same as a `readDir path`, so we can turn it back into an equivalent "directory".
      else if all isString subtreeValues then
        "directory"
      else
        normalisedSubtrees
    else
      tree;

  /*
    A minimal normalisation of a filesetTree, intended for pretty-printing:
    - If all children of a path are recursively included or empty directories, the path itself is also recursively included
    - If all children of a path are fully excluded or empty directories, the path itself is an empty directory
    - Other empty directories are represented with the special "emptyDir" string
      While these could be replaced with `null`, that would take another mapAttrs

    Note that this function is partially lazy.

    Type: Path -> filesetTree -> filesetTree (with "emptyDir"'s)
  */
  _normaliseTreeMinimal = path: tree:
    if tree == "directory" || isAttrs tree then
      let
        entries = _directoryEntries path tree;
        normalisedSubtrees = mapAttrs (name: _normaliseTreeMinimal (path + "/${name}")) entries;
        subtreeValues = attrValues normalisedSubtrees;
      in
      # If there are no entries, or all entries are empty directories, return "emptyDir".
      # After this branch we know that there's at least one file
      if all (value: value == "emptyDir") subtreeValues then
        "emptyDir"

      # If all subtrees are fully included or empty directories
      # (both of which are coincidentally represented as strings), return "directory".
      # This takes advantage of the fact that empty directories can be represented as included directories.
      # Note that the tree == "directory" check allows avoiding recursion
      else if tree == "directory" || all (value: isString value) subtreeValues then
        "directory"

      # If all subtrees are fully excluded or empty directories, return null.
      # This takes advantage of the fact that empty directories can be represented as excluded directories
      else if all (value: isNull value || value == "emptyDir") subtreeValues then
        null

      # Mix of included and excluded entries
      else
        normalisedSubtrees
    else
      tree;

  # Trace a filesetTree in a pretty way when the resulting value is evaluated.
  # This can handle both normal filesetTree's, and ones returned from _normaliseTreeMinimal
  # Type: Path -> filesetTree (with "emptyDir"'s) -> Null
  _printMinimalTree = base: tree:
    let
      treeSuffix = tree:
        if isAttrs tree then
          ""
        else if tree == "directory" then
          " (all files in directory)"
        else
          # This does "leak" the file type strings of the internal representation,
          # but this is the main reason these file type strings even are in the representation!
          # TODO: Consider removing that information from the internal representation for performance.
          # The file types can still be printed by querying them only during tracing
          " (${tree})";

      # Only for attribute set trees
      traceTreeAttrs = prevLine: indent: tree:
        foldl' (prevLine: name:
          let
            subtree = tree.${name};

            # Evaluating this prints the line for this subtree
            thisLine =
              trace "${indent}- ${name}${treeSuffix subtree}" prevLine;
          in
          if subtree == null || subtree == "emptyDir" then
            # Don't print anything at all if this subtree is empty
            prevLine
          else if isAttrs subtree then
            # A directory with explicit entries
            # Do print this node, but also recurse
            traceTreeAttrs thisLine "${indent}  " subtree
          else
            # Either a file, or a recursively included directory
            # Do print this node but no further recursion needed
            thisLine
        ) prevLine (attrNames tree);

      # Evaluating this will print the first line
      firstLine =
        if tree == null || tree == "emptyDir" then
          trace "(empty)" null
        else
          trace "${toString base}${treeSuffix tree}" null;
    in
    if isAttrs tree then
      traceTreeAttrs firstLine "" tree
    else
      firstLine;

  # Pretty-print a file set in a pretty way when the resulting value is evaluated
  # Type: fileset -> Null
  _printFileset = fileset:
    if fileset._internalIsEmptyWithoutBase then
      trace "(empty)" null
    else
      _printMinimalTree fileset._internalBase
        (_normaliseTreeMinimal fileset._internalBase fileset._internalTree);

  # Turn a fileset into a source filter function suitable for `builtins.path`
  # Only directories recursively containing at least one files are recursed into
  # Type: Path -> fileset -> (String -> String -> Bool)
  _toSourceFilter = fileset:
    let
      # Simplify the tree, necessary to make sure all empty directories are null
      # which has the effect that they aren't included in the result
      tree = _normaliseTreeFilter fileset._internalBase fileset._internalTree;

      # The base path as a string with a single trailing slash
      baseString =
        if fileset._internalBaseComponents == [] then
          # Need to handle the filesystem root specially
          "/"
        else
          "/" + concatStringsSep "/" fileset._internalBaseComponents + "/";

      baseLength = stringLength baseString;

      # Check whether a list of path components under the base path exists in the tree.
      # This function is called often, so it should be fast.
      # Type: [ String ] -> Bool
      inTree = components:
        let
          recurse = index: localTree:
            if isAttrs localTree then
              # We have an attribute set, meaning this is a directory with at least one file
              if index >= length components then
                # The path may have no more components though, meaning the filter is running on the directory itself,
                # so we always include it, again because there's at least one file in it.
                true
              else
                # If we do have more components, the filter runs on some entry inside this directory, so we need to recurse
                # We do +2 because builtins.split is an interleaved list of the inbetweens and the matches
                recurse (index + 2) localTree.${elemAt components index}
            else
              # If it's not an attribute set it can only be either null (in which case it's not included)
              # or a string ("directory" or "regular", etc.) in which case it's included
              localTree != null;
        in recurse 0 tree;

      # Filter suited when there's no files
      empty = _: _: false;

      # Filter suited when there's some files
      # This can't be used for when there's no files, because the base directory is always included
      nonEmpty =
        path: type:
        let
          # Add a slash to the path string, turning "/foo" to "/foo/",
          # making sure to not have any false prefix matches below.
          # Note that this would produce "//" for "/",
          # but builtins.path doesn't call the filter function on the `path` argument itself,
          # meaning this function can never receive "/" as an argument
          pathSlash = path + "/";

          include =
            # Same as `hasPrefix pathSlash baseString`, but more efficient.
            # With base /foo/bar we need to include /foo:
            # hasPrefix "/foo/" "/foo/bar/"
            if substring 0 (stringLength pathSlash) baseString == pathSlash then
              true
            # Same as `! hasPrefix baseString pathSlash`, but more efficient.
            # With base /foo/bar we need to exclude /baz
            # ! hasPrefix "/baz/" "/foo/bar/"
            else if substring 0 baseLength pathSlash != baseString then
              false
            else
              # Same as `removePrefix baseString path`, but more efficient.
              # From the above code we know that hasPrefix baseString pathSlash holds, so this is safe.
              # We don't use pathSlash here because we only needed the trailing slash for the prefix matching.
              # With base /foo and path /foo/bar/baz this gives
              # inTree (split "/" (removePrefix "/foo/" "/foo/bar/baz"))
              # == inTree (split "/" "bar/baz")
              # == inTree [ "bar" "baz" ]
              inTree (split "/" (substring baseLength (-1) path));
        in
        # This relies on the fact that Nix only distinguishes path types "directory", "regular", "symlink" and "unknown",
        # so everything except "unknown" is allowed, seems reasonable to rely on that
        if include && type == "unknown" then
          throw ''
            lib.fileset.toSource: `fileset` contains a file that cannot be added to the store: ${path}
                This file is neither a regular file nor a symlink, the only file types supported by the Nix store.
                Therefore the file set cannot be added to the Nix store as is. Make sure to not include that file to avoid this error.''
        else
          include;
    in
    # Special case because the code below assumes that the _internalBase is always included in the result
    # which shouldn't be done when we have no files at all in the base
    # This also forces the tree before returning the filter, leads to earlier error messages
    if fileset._internalIsEmptyWithoutBase || tree == null then
      empty
    else
      nonEmpty;

  # Transforms the filesetTree of a file set to a shorter base path, e.g.
  # _shortenTreeBase [ "foo" ] (_create /foo/bar null)
  # => { bar = null; }
  _shortenTreeBase = targetBaseComponents: fileset:
    let
      recurse = index:
        # If we haven't reached the required depth yet
        if index < length fileset._internalBaseComponents then
          # Create an attribute set and recurse as the value, this can be lazily evaluated this way
          { ${elemAt fileset._internalBaseComponents index} = recurse (index + 1); }
        else
          # Otherwise we reached the appropriate depth, here's the original tree
          fileset._internalTree;
    in
    recurse (length targetBaseComponents);

  # Transforms the filesetTree of a file set to a longer base path, e.g.
  # _lengthenTreeBase [ "foo" "bar" ] (_create /foo { bar.baz = "regular"; })
  # => { baz = "regular"; }
  _lengthenTreeBase = targetBaseComponents: fileset:
    let
      recurse = index: tree:
        # If the filesetTree is an attribute set and we haven't reached the required depth yet
        if isAttrs tree && index < length targetBaseComponents then
          # Recurse with the tree under the right component (which might not exist)
          recurse (index + 1) (tree.${elemAt targetBaseComponents index} or null)
        else
          # For all values here we can just return the tree itself:
          # tree == null -> the result is also null, everything is excluded
          # tree == "directory" -> the result is also "directory",
          #   because the base path is always a directory and everything is included
          # isAttrs tree -> the result is `tree`
          #   because we don't need to recurse any more since `index == length longestBaseComponents`
          tree;
    in
    recurse (length fileset._internalBaseComponents) fileset._internalTree;

  # Computes the union of a list of filesets.
  # The filesets must already be coerced and validated to be in the same filesystem root
  # Type: [ Fileset ] -> Fileset
  _unionMany = filesets:
    let
      # All filesets that have a base, aka not the ones that are the empty value without a base
      filesetsWithBase = filter (fileset: ! fileset._internalIsEmptyWithoutBase) filesets;

      # The first fileset that has a base.
      # This value is only accessed if there are at all.
      firstWithBase = head filesetsWithBase;

      # To be able to union filesetTree's together, they need to have the same base path.
      # Base paths can be unioned by taking their common prefix,
      # e.g. such that `union /foo/bar /foo/baz` has the base path `/foo`

      # A list of path components common to all base paths.
      # Note that commonPrefix can only be fully evaluated,
      # so this cannot cause a stack overflow due to a build-up of unevaluated thunks.
      commonBaseComponents = foldl'
        (components: el: commonPrefix components el._internalBaseComponents)
        firstWithBase._internalBaseComponents
        # We could also not do the `tail` here to avoid a list allocation,
        # but then we'd have to pay for a potentially expensive
        # but unnecessary `commonPrefix` call
        (tail filesetsWithBase);

      # The common base path assembled from a filesystem root and the common components
      commonBase = append firstWithBase._internalBaseRoot (join commonBaseComponents);

      # A list of filesetTree's that all have the same base path
      # This is achieved by nesting the trees into the components they have over the common base path
      # E.g. `union /foo/bar /foo/baz` has the base path /foo
      # So the tree under `/foo/bar` gets nested under `{ bar = ...; ... }`,
      # while the tree under `/foo/baz` gets nested under `{ baz = ...; ... }`
      # Therefore allowing combined operations over them.
      trees = map (_shortenTreeBase commonBaseComponents) filesetsWithBase;

      # Folds all trees together into a single one using _unionTree
      # We do not use a fold here because it would cause a thunk build-up
      # which could cause a stack overflow for a large number of trees
      resultTree = _unionTrees trees;
    in
    # If there's no values with a base, we have no files
    if filesetsWithBase == [ ] then
      _emptyWithoutBase
    else
      _create commonBase resultTree;

  # The union of multiple filesetTree's with the same base path.
  # Later elements are only evaluated if necessary.
  # Type: [ filesetTree ] -> filesetTree
  _unionTrees = trees:
    let
      stringIndex = findFirstIndex isString null trees;
      withoutNull = filter (tree: tree != null) trees;
    in
    if stringIndex != null then
      # If there's a string, it's always a fully included tree (dir or file),
      # no need to look at other elements
      elemAt trees stringIndex
    else if withoutNull == [ ] then
      # If all trees are null, then the resulting tree is also null
      null
    else
      # The non-null elements have to be attribute sets representing partial trees
      # We need to recurse into those
      zipAttrsWith (name: _unionTrees) withoutNull;

  # Computes the intersection of a list of filesets.
  # The filesets must already be coerced and validated to be in the same filesystem root
  # Type: Fileset -> Fileset -> Fileset
  _intersection = fileset1: fileset2:
    let
      # The common base components prefix, e.g.
      # (/foo/bar, /foo/bar/baz) -> /foo/bar
      # (/foo/bar, /foo/baz) -> /foo
      commonBaseComponentsLength =
        # TODO: Have a `lib.lists.commonPrefixLength` function such that we don't need the list allocation from commonPrefix here
        length (
          commonPrefix
            fileset1._internalBaseComponents
            fileset2._internalBaseComponents
        );

      # To be able to intersect filesetTree's together, they need to have the same base path.
      # Base paths can be intersected by taking the longest one (if any)

      # The fileset with the longest base, if any, e.g.
      # (/foo/bar, /foo/bar/baz) -> /foo/bar/baz
      # (/foo/bar, /foo/baz) -> null
      longestBaseFileset =
        if commonBaseComponentsLength == length fileset1._internalBaseComponents then
          # The common prefix is the same as the first path, so the second path is equal or longer
          fileset2
        else if commonBaseComponentsLength == length fileset2._internalBaseComponents then
          # The common prefix is the same as the second path, so the first path is longer
          fileset1
        else
          # The common prefix is neither the first nor the second path
          # This means there's no overlap between the two sets
          null;

      # Whether the result should be the empty value without a base
      resultIsEmptyWithoutBase =
        # If either fileset is the empty fileset without a base, the intersection is too
        fileset1._internalIsEmptyWithoutBase
        || fileset2._internalIsEmptyWithoutBase
        # If there is no overlap between the base paths
        || longestBaseFileset == null;

      # Lengthen each fileset's tree to the longest base prefix
      tree1 = _lengthenTreeBase longestBaseFileset._internalBaseComponents fileset1;
      tree2 = _lengthenTreeBase longestBaseFileset._internalBaseComponents fileset2;

      # With two filesetTree's with the same base, we can compute their intersection
      resultTree = _intersectTree tree1 tree2;
    in
    if resultIsEmptyWithoutBase then
      _emptyWithoutBase
    else
      _create longestBaseFileset._internalBase resultTree;

  # The intersection of two filesetTree's with the same base path
  # The second element is only evaluated as much as necessary.
  # Type: filesetTree -> filesetTree -> filesetTree
  _intersectTree = lhs: rhs:
    if isAttrs lhs && isAttrs rhs then
      # Both sides are attribute sets, we can recurse for the attributes existing on both sides
      mapAttrs
        (name: _intersectTree lhs.${name})
        (builtins.intersectAttrs lhs rhs)
    else if lhs == null || isString rhs then
      # If the lhs is null, the result should also be null
      # And if the rhs is the identity element
      # (a string, aka it includes everything), then it's also the lhs
      lhs
    else
      # In all other cases it's the rhs
      rhs;

  _fileFilter = predicate: fileset:
    let
      recurse = path: tree:
        mapAttrs (name: subtree:
          if isAttrs subtree || subtree == "directory" then
            recurse (path + "/${name}") subtree
          else if
            predicate {
              inherit name;
              type = subtree;
              # To ensure forwards compatibility with more arguments being added in the future,
              # adding an attribute which can't be deconstructed :)
              "lib.fileset.fileFilter: The predicate function passed as the first argument must be able to handle extra attributes for future compatibility. If you're using `{ name, file }:`, use `{ name, file, ... }:` instead." = null;
            }
          then
            subtree
          else
            null
        ) (_directoryEntries path tree);
    in
    if fileset._internalIsEmptyWithoutBase then
      _emptyWithoutBase
    else
      _create fileset._internalBase
        (recurse fileset._internalBase fileset._internalTree);
}