Merge pull request #223104 from xworld21/texlive-fixed-hashes-nix

texlive: generate "fixed-hashes.nix" from Nixpkgs attribute

pkgs/tools/typesetting/tex/texlive/UPGRADING.md

@@ -45,24 +45,39 @@ in Nixpkgs matches the one that generated from `texlive.tlpdb.xz`.
 
 ### Build packages locally and generate fix hashes
 
-To save disk space and prevent unnecessary rebuilds, texlive packages are built
-as fixed-output derivations whose hashes are contained in `fixedHashes.nix`.
-
-Updating the list of fixed hashes requires a local build of *all* packages,
-which is a resource-intensive process:
+To prevent unnecessary rebuilds, texlive packages are built as fixed-output
+derivations whose hashes are contained in `fixed-hashes.nix`.
 
+Updating the list of fixed hashes requires a local build of all new packages,
+which is a resource-intensive process. First build the hashes for the new
+packages. Consider tweaking the `-j` option to maximise core usage.
 
 ```bash
-# move fixedHashes away, otherwise build will fail on updated packages
-mv fixedHashes.nix fixedHashes-old.nix
-# start with empty fixedHashes
-echo '{}' > fixedHashes.nix
-
-nix-build ../../../../.. -Q --no-out-link -A texlive.scheme-full.pkgs | ./fixHashes.awk > ./fixedHashes-new.nix
-
-mv fixedHashes-new.nix fixedHashes.nix
+nix-build generate-fixed-hashes.nix -A newHashes -j 8
 ```
 
+Then build the Nix expression containing all the hashes, old and new. This step
+cannot be parallelized because it relies on 'import from derivation'.
+
+```bash
+nix-build generate-fixed-hashes.nix -A fixedHashesNix
+```
+
+Finally, copy the result to `fixed-hashes.nix`.
+
+**Warning.** The expression `fixedHashesNix` reuses the *previous* fixed hashes
+when possible. This is based on two assumptions: that `.tar.xz` archives with
+the same names remain identical in time (which is the intended behaviour of
+CTAN and the various mirrors) and that the build recipe continues to produce
+the same output. Should those assumptions not hold, remove the previous fixed
+hashes for the relevant package, or for all packages.
+
 ### Commit changes
 
-Commit the updated `tlpdb.nix` and `fixedHashes.nix` to the repository.
+Commit the updated `tlpdb.nix` and `fixed-hashes.nix` to the repository with
+a message like
+
+> texlive: 2022-final -> 2023.20230401
+
+Please make sure to follow the [CONTRIBUTING](https://github.com/NixOS/nixpkgs/blob/master/CONTRIBUTING.md)
+guidelines.

pkgs/tools/typesetting/tex/texlive/bin.nix

@@ -5,7 +5,7 @@
 , perl, perlPackages, python3Packages, pkg-config
 , libpaper, graphite2, zziplib, harfbuzz, potrace, gmp, mpfr
 , brotli, cairo, pixman, xorg, clisp, biber, woff2, xxHash
-, makeWrapper, shortenPerlShebang
+, makeWrapper, shortenPerlShebang, useFixedHashes
 }:
 
 # Useful resource covering build options:
@@ -17,6 +17,10 @@ let
   year = toString ((import ./tlpdb.nix)."00texlive.config").year;
   version = year; # keep names simple for now
 
+  # detect and stop redundant rebuilds that may occur when building new fixed hashes
+  assertFixedHash = name: src:
+    if ! useFixedHashes || src ? outputHash then src else throw "The TeX Live package '${src.pname}' must have a fixed hash before building '${name}'.";
+
   common = {
     src = fetchurl {
       urls = [
@@ -369,7 +373,7 @@ latexindent = perlPackages.buildPerlPackage rec {
   pname = "latexindent";
   inherit (src) version;
 
-  src = lib.head (builtins.filter (p: p.tlType == "run") texlive.latexindent.pkgs);
+  src = assertFixedHash pname (lib.head (builtins.filter (p: p.tlType == "run") texlive.latexindent.pkgs));
 
   outputs = [ "out" ];
 
@@ -401,7 +405,7 @@ pygmentex = python3Packages.buildPythonApplication rec {
   inherit (src) version;
   format = "other";
 
-  src = lib.head (builtins.filter (p: p.tlType == "run") texlive.pygmentex.pkgs);
+  src = assertFixedHash pname (lib.head (builtins.filter (p: p.tlType == "run") texlive.pygmentex.pkgs));
 
   propagatedBuildInputs = with python3Packages; [ pygments chardet ];
 
@@ -437,7 +441,7 @@ pygmentex = python3Packages.buildPythonApplication rec {
 texlinks = stdenv.mkDerivation rec {
   name = "texlinks";
 
-  src = lib.head (builtins.filter (p: p.tlType == "run") texlive.texlive-scripts-extra.pkgs);
+  src = assertFixedHash name (lib.head (builtins.filter (p: p.tlType == "run") texlive.texlive-scripts-extra.pkgs));
 
   dontBuild = true;
   doCheck = false;

pkgs/tools/typesetting/tex/texlive/default.nix

@@ -16,11 +16,9 @@ let
     harfbuzz = harfbuzz.override {
       withIcu = true; withGraphite2 = true;
     };
+    inherit useFixedHashes;
   };
 
-  # map: name -> fixed-output hash
-  fixedHashes = lib.optionalAttrs useFixedHashes (import ./fixedHashes.nix);
-
   # function for creating a working environment from a set of TL packages
   combine = import ./combine.nix {
     inherit bin combinePkgs buildEnv lib makeWrapper writeText runCommand
@@ -59,7 +57,8 @@ let
       };
 
       texdoc = orig.texdoc // {
-        version = orig.texdoc.version + "-tlpdb-" + (toString tlpdbVersion.revision);
+        extraRevision = ".tlpdb${toString tlpdbVersion.revision}";
+        extraVersion = "-tlpdb-${toString tlpdbVersion.revision}";
 
         # build Data.tlpdb.lua (part of the 'tlType == "run"' package)
         postUnpack = ''
@@ -155,13 +154,24 @@ let
     xzcat "$tlpdbxz" | sed -rn -f "$tl2nix" | uniq > "$out"
   '';
 
+  # map: name -> fixed-output hash
+  fixedHashes = lib.optionalAttrs useFixedHashes (import ./fixed-hashes.nix);
+
+  # NOTE: the fixed naming scheme must match generated-fixed-hashes.nix
+  # name for the URL
+  mkURLName = { pname, tlType, ... }: pname + lib.optionalString (tlType != "run" && tlType != "tlpkg") ".${tlType}";
+  # name + revision for the fixed output hashes
+  mkFixedName = { tlType, revision, extraRevision ? "", ... }@attrs: mkURLName attrs + (lib.optionalString (tlType == "tlpkg") ".tlpkg") + ".r${toString revision}${extraRevision}";
+  # name + version for the derivation
+  mkTLName = { tlType, version, extraVersion ? "", ... }@attrs: mkURLName attrs + (lib.optionalString (tlType == "tlpkg") ".tlpkg") + "-${version}${extraVersion}";
+
   # create a derivation that contains an unpacked upstream TL package
-  mkPkg = { pname, tlType, revision, version, sha512, postUnpack ? "", stripPrefix ? 1, ... }@args:
+  mkPkg = { pname, tlType, revision, version, sha512, extraRevision ? "", postUnpack ? "", stripPrefix ? 1, ... }@args:
     let
       # the basename used by upstream (without ".tar.xz" suffix)
-      urlName = pname + lib.optionalString (tlType != "run" && tlType != "tlpkg") ".${tlType}";
-      tlName = urlName + lib.optionalString (tlType == "tlpkg") ".tlpkg" + "-${version}";
-      fixedHash = fixedHashes.${tlName} or null; # be graceful about missing hashes
+      urlName = mkURLName args;
+      tlName = mkTLName args;
+      fixedHash = fixedHashes.${mkFixedName args} or null; # be graceful about missing hashes
 
       urls = args.urls or (if args ? url then [ args.url ] else
         map (up: "${up}/archive/${urlName}.r${toString revision}.tar.xz") (args.urlPrefixes or urlPrefixes));
@@ -172,7 +182,7 @@ let
           inherit stripPrefix tlType;
           # metadata for texlive.combine
           passthru = {
-            inherit pname tlType version;
+            inherit pname tlType revision version extraRevision;
           } // lib.optionalAttrs (tlType == "run" && args ? deps) {
             tlDeps = map (n: tl.${n}) args.deps;
           } // lib.optionalAttrs (tlType == "run") {
@@ -218,9 +228,13 @@ let
       operator = { pkg, ... }: pkgListToSets (pkg.tlDeps or []);
     });
 
-  assertions =
-    lib.assertMsg (tlpdbVersion.year == version.texliveYear) "TeX Live year in texlive does not match tlpdb.nix, refusing to evaluate" &&
-    lib.assertMsg (tlpdbVersion.frozen == version.final) "TeX Live final status in texlive does not match tlpdb.nix, refusing to evaluate";
+  assertions = with lib;
+    assertMsg (tlpdbVersion.year == version.texliveYear) "TeX Live year in texlive does not match tlpdb.nix, refusing to evaluate" &&
+    assertMsg (tlpdbVersion.frozen == version.final) "TeX Live final status in texlive does not match tlpdb.nix, refusing to evaluate" &&
+    (!useFixedHashes ||
+      (let all = concatLists (catAttrs "pkgs" (attrValues tl));
+         fods = filter (p: isDerivation p && p.tlType != "bin") all;
+      in builtins.all (p: assertMsg (p ? outputHash) "The TeX Live package '${p.pname + lib.optionalString (p.tlType != "run") ("." + p.tlType)}' does not have a fixed output hash. Please read UPGRADING.md on how to build a new 'fixed-hashes.nix'.") fods));
 
 in
   tl // {

pkgs/tools/typesetting/tex/texlive/fixHashes.awk

@@ -1,24 +0,0 @@
-#!/usr/bin/env nix-shell
-#! nix-shell -i "gawk -f" -p gawk
-
-BEGIN {
-  print "{"
-}
-
-/-texlive-/ && !/\.bin/ {
-  if (match($0, /-texlive-([^\/]*)/, m) == 0) {
-    print "No match for \""$0"\"" > "/dev/stderr"
-    exit 1
-  }
-  cmd="nix-hash --type sha256 --base32 "$0
-  if (( cmd | getline hash ) <= 0) {
-    print "Error executing nix-hash" > "/dev/stderr"
-    exit 1
-  }
-  close(cmd)
-  printf("\"%s\"=\"%s\";\n", m[1], hash)
-}
-
-END {
-  print "}"
-}

pkgs/tools/typesetting/tex/texlive/generate-fixed-hashes.nix

@@ -0,0 +1,41 @@
+with import ../../../../.. { };
+
+with lib; let
+  # NOTE: the fixed naming scheme must match default.nix
+  # name for the URL
+  mkURLName = { pname, tlType, ... }: pname + lib.optionalString (tlType != "run" && tlType != "tlpkg") ".${tlType}";
+  # name + revision for the fixed output hashes
+  mkFixedName = { tlType, revision, extraRevision ? "", ... }@attrs: mkURLName attrs + (lib.optionalString (tlType == "tlpkg") ".tlpkg") + ".r${toString revision}${extraRevision}";
+
+  uniqueByName = fods: catAttrs "fod" (genericClosure {
+    startSet = map (fod: { key = fod.name; inherit fod; }) fods;
+    operator = _: [ ];
+  });
+
+  # ugly hack to extract combine from collection-latexextra, since it is masked by texlive.combine
+  combine = lib.findFirst (p: (lib.head p.pkgs).pname == "combine") { pkgs = []; } (lib.head texlive.collection-latexextra.pkgs).tlDeps;
+  all = concatLists (map (p: p.pkgs or []) (attrValues (removeAttrs texlive [ "bin" "combine" "combined" "tlpdb" ]))) ++ combine.pkgs;
+
+  # fixed hashes only for run, doc, source, tlpkg types
+  fods = sort (a: b: a.name < b.name) (uniqueByName (filter (p: isDerivation p && p.tlType != "bin") all));
+
+  computeHash = fod: runCommand "${fod.name}-fixed-hash"
+    { buildInputs = [ nix ]; inherit fod; }
+    ''echo -n "$(nix-hash --base32 --type sha256 "$fod")" >"$out"'';
+
+  hash = fod: fod.outputHash or (builtins.readFile (computeHash fod));
+  hashLine = fod: ''
+    "${mkFixedName fod}"="${hash fod}";
+  '';
+in
+{
+  # fixedHashesNix uses 'import from derivation' which does not parallelize well
+  # you should build newHashes first, before evaluating (and building) fixedHashesNix
+  newHashes = map computeHash (filter (fod: ! fod ? outputHash) fods);
+
+  fixedHashesNix = writeText "fixed-hashes.nix"
+  ''
+    {
+    ${lib.concatMapStrings hashLine fods}}
+  '';
+}