From 59a94b57f07594f4544896dd90c71a948d1ea089 Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Fri, 23 Nov 2018 18:03:19 +0100 Subject: [PATCH] update.nix: Run update scripts in parallel To make updating large attribute sets faster, the update scripts are now run in parallel. Please note the following changes in semantics: - The string passed to updateScript needs to be a path to an executable file. - The updateScript can also be a list: the tail elements will then be passed to the head as command line arguments. --- doc/stdenv.xml | 37 +++++++++ maintainers/scripts/update.nix | 53 +++++-------- maintainers/scripts/update.py | 79 +++++++++++++++++++ .../browsers/firefox-bin/default.nix | 2 +- .../browsers/firefox-bin/update.nix | 4 +- .../networking/browsers/firefox/update.nix | 2 + pkgs/applications/office/mendeley/default.nix | 2 +- pkgs/applications/office/mendeley/update.nix | 3 +- pkgs/desktops/deepin/update.nix | 3 +- pkgs/desktops/gnome-3/update.nix | 3 +- .../libraries/safefile/default.nix | 1 + .../tools/build-managers/gup/default.nix | 1 + pkgs/development/web/nodejs/update.nix | 4 +- pkgs/os-specific/linux/tp_smapi/default.nix | 2 +- pkgs/os-specific/linux/tp_smapi/update.nix | 3 +- .../package-management/nix-pin/default.nix | 1 + .../nix-update-source/default.nix | 3 +- 17 files changed, 159 insertions(+), 44 deletions(-) create mode 100644 maintainers/scripts/update.py diff --git a/doc/stdenv.xml b/doc/stdenv.xml index 208b5e9cf302..a90a377b3f26 100644 --- a/doc/stdenv.xml +++ b/doc/stdenv.xml @@ -671,6 +671,43 @@ passthru = { + + + passthru.updateScript + + + + A script to be run by maintainers/scripts/update.nix when + the package is matched. It needs to be an executable file, either on the file + system: + +passthru.updateScript = ./update.sh; + + or inside the expression itself: + +passthru.updateScript = writeScript "update-zoom-us" '' + #!/usr/bin/env nix-shell + #!nix-shell -i bash -p curl pcre common-updater-scripts + + set -eu -o pipefail + + version="$(curl -sI https://zoom.us/client/latest/zoom_x86_64.tar.xz | grep -Fi 'Location:' | pcregrep -o1 '/(([0-9]\.?)+)/')" + update-source-version zoom-us "$version" +''; + + The attribute can also contain a list, a script followed by arguments to be passed to it: + +passthru.updateScript = [ ../../update.sh pname "--requested-release=unstable" ]; + + Note that the update scripts will be run in parallel by default; you should avoid running git commit or any other commands that cannot handle that. + + + + For information about how to run the updates, execute + nix-shell maintainers/scripts/update.nix. + + +
diff --git a/maintainers/scripts/update.nix b/maintainers/scripts/update.nix index 8d1e47c6bc95..120cd5552f48 100755 --- a/maintainers/scripts/update.nix +++ b/maintainers/scripts/update.nix @@ -1,6 +1,8 @@ { package ? null , maintainer ? null , path ? null +, max-workers ? null +, keep-going ? null }: # TODO: add assert statements @@ -105,26 +107,23 @@ let % nix-shell maintainers/scripts/update.nix --argstr path gnome3 to run update script for all package under an attribute path. + + You can also add + + --argstr max-workers 8 + + to increase the number of jobs in parallel, or + + --argstr keep-going true + + to continue running when a single update fails. ''; - runUpdateScript = package: '' - echo -ne " - ${package.name}: UPDATING ..."\\r - ${package.updateScript} &> ${(builtins.parseDrvName package.name).name}.log - CODE=$? - if [ "$CODE" != "0" ]; then - echo " - ${package.name}: ERROR " - echo "" - echo "--- SHOWING ERROR LOG FOR ${package.name} ----------------------" - echo "" - cat ${(builtins.parseDrvName package.name).name}.log - echo "" - echo "--- SHOWING ERROR LOG FOR ${package.name} ----------------------" - exit $CODE - else - rm ${(builtins.parseDrvName package.name).name}.log - fi - echo " - ${package.name}: DONE. " - ''; + packageData = package: { + name = package.name; + pname = (builtins.parseDrvName package.name).name; + updateScript = pkgs.lib.toList package.updateScript; + }; in pkgs.stdenv.mkDerivation { name = "nixpkgs-update-script"; @@ -139,21 +138,7 @@ in pkgs.stdenv.mkDerivation { exit 1 ''; shellHook = '' - echo "" - echo "Going to be running update for following packages:" - echo "${builtins.concatStringsSep "\n" (map (x: " - ${x.name}") packages)}" - echo "" - read -n1 -r -p "Press space to continue..." confirm - if [ "$confirm" = "" ]; then - echo "" - echo "Running update for:" - ${builtins.concatStringsSep "\n" (map runUpdateScript packages)} - echo "" - echo "Packages updated!" - exit 0 - else - echo "Aborting!" - exit 1 - fi + unset shellHook # do not contaminate nested shells + exec ${pkgs.python3.interpreter} ${./update.py} ${pkgs.writeText "packages.json" (builtins.toJSON (map packageData packages))}${pkgs.lib.optionalString (max-workers != null) " --max-workers=${max-workers}"}${pkgs.lib.optionalString (keep-going == "true") " --keep-going"} ''; } diff --git a/maintainers/scripts/update.py b/maintainers/scripts/update.py new file mode 100644 index 000000000000..eb7d0ef2647b --- /dev/null +++ b/maintainers/scripts/update.py @@ -0,0 +1,79 @@ +import argparse +import concurrent.futures +import json +import os +import subprocess +import sys + +updates = {} + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + +def run_update_script(package): + eprint(f" - {package['name']}: UPDATING ...") + + subprocess.run(package['updateScript'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True) + + +def main(max_workers, keep_going, packages): + with open(sys.argv[1]) as f: + packages = json.load(f) + + eprint() + eprint('Going to be running update for following packages:') + for package in packages: + eprint(f" - {package['name']}") + eprint() + + confirm = input('Press Enter key to continue...') + if confirm == '': + eprint() + eprint('Running update for:') + + with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor: + for package in packages: + updates[executor.submit(run_update_script, package)] = package + + for future in concurrent.futures.as_completed(updates): + package = updates[future] + + try: + future.result() + eprint(f" - {package['name']}: DONE.") + except subprocess.CalledProcessError as e: + eprint(f" - {package['name']}: ERROR") + eprint() + eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------") + eprint() + eprint(e.stdout.decode('utf-8')) + with open(f"{package['pname']}.log", 'wb') as f: + f.write(e.stdout) + eprint() + eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------") + + if not keep_going: + sys.exit(1) + + eprint() + eprint('Packages updated!') + sys.exit() + else: + eprint('Aborting!') + sys.exit(130) + +parser = argparse.ArgumentParser(description='Update packages') +parser.add_argument('--max-workers', '-j', dest='max_workers', type=int, help='Number of updates to run concurrently', nargs='?', default=4) +parser.add_argument('--keep-going', '-k', dest='keep_going', action='store_true', help='Do not stop after first failure') +parser.add_argument('packages', help='JSON file containing the list of package names and their update scripts') + +if __name__ == '__main__': + args = parser.parse_args() + + try: + main(args.max_workers, args.keep_going, args.packages) + except (KeyboardInterrupt, SystemExit) as e: + for update in updates: + update.cancel() + + sys.exit(e.code if isinstance(e, SystemExit) else 130) diff --git a/pkgs/applications/networking/browsers/firefox-bin/default.nix b/pkgs/applications/networking/browsers/firefox-bin/default.nix index c81c79349857..2d9692c0528e 100644 --- a/pkgs/applications/networking/browsers/firefox-bin/default.nix +++ b/pkgs/applications/networking/browsers/firefox-bin/default.nix @@ -191,7 +191,7 @@ stdenv.mkDerivation { # update with: # $ nix-shell maintainers/scripts/update.nix --argstr package firefox-bin-unwrapped passthru.updateScript = import ./update.nix { - inherit name channel writeScript xidel coreutils gnused gnugrep gnupg curl; + inherit stdenv name channel writeScript xidel coreutils gnused gnugrep gnupg curl; baseUrl = if channel == "devedition" then "http://archive.mozilla.org/pub/devedition/releases/" diff --git a/pkgs/applications/networking/browsers/firefox-bin/update.nix b/pkgs/applications/networking/browsers/firefox-bin/update.nix index df928f88c386..ee022e329f9f 100644 --- a/pkgs/applications/networking/browsers/firefox-bin/update.nix +++ b/pkgs/applications/networking/browsers/firefox-bin/update.nix @@ -1,4 +1,5 @@ -{ name +{ stdenv +, name , channel , writeScript , xidel @@ -17,6 +18,7 @@ let channel != "release"; in writeScript "update-${name}" '' + #!${stdenv.shell} PATH=${coreutils}/bin:${gnused}/bin:${gnugrep}/bin:${xidel}/bin:${curl}/bin:${gnupg}/bin set -eux pushd ${basePath} diff --git a/pkgs/applications/networking/browsers/firefox/update.nix b/pkgs/applications/networking/browsers/firefox/update.nix index a831d8231187..07ae2c040e6e 100644 --- a/pkgs/applications/networking/browsers/firefox/update.nix +++ b/pkgs/applications/networking/browsers/firefox/update.nix @@ -1,4 +1,5 @@ { writeScript +, stdenv , lib , xidel , common-updater-scripts @@ -13,6 +14,7 @@ }: writeScript "update-${attrPath}" '' + #!${stdenv.shell} PATH=${lib.makeBinPath [ common-updater-scripts coreutils curl gnugrep gnused xidel ]} url=${baseUrl} diff --git a/pkgs/applications/office/mendeley/default.nix b/pkgs/applications/office/mendeley/default.nix index 5b6271db83bd..aa9317d2ffd6 100644 --- a/pkgs/applications/office/mendeley/default.nix +++ b/pkgs/applications/office/mendeley/default.nix @@ -131,7 +131,7 @@ stdenv.mkDerivation { dontStrip = true; dontPatchElf = true; - updateScript = import ./update.nix { inherit writeScript; }; + updateScript = import ./update.nix { inherit stdenv writeScript; }; meta = with stdenv.lib; { homepage = http://www.mendeley.com; diff --git a/pkgs/applications/office/mendeley/update.nix b/pkgs/applications/office/mendeley/update.nix index cb9ee02702dd..147c95b8e7bd 100644 --- a/pkgs/applications/office/mendeley/update.nix +++ b/pkgs/applications/office/mendeley/update.nix @@ -1,6 +1,7 @@ -{ writeScript }: +{ stdenv, writeScript }: writeScript "update-mendeley" '' + #!${stdenv.shell} function follow() { local URL=$1 while true; do diff --git a/pkgs/desktops/deepin/update.nix b/pkgs/desktops/deepin/update.nix index 761ead015c6b..22a6acb8ce31 100644 --- a/pkgs/desktops/deepin/update.nix +++ b/pkgs/desktops/deepin/update.nix @@ -1,4 +1,4 @@ -{ lib, writeScript, coreutils, curl, gnugrep, gnused, jq, common-updater-scripts, nix }: +{ stdenv, lib, writeScript, coreutils, curl, gnugrep, gnused, jq, common-updater-scripts, nix }: { name, ignored-versions ? "^2014\\.|^v[0-9]+" }: let @@ -9,6 +9,7 @@ let in writeScript "update-${packageName}" '' + #!${stdenv.shell} set -o errexit set -x diff --git a/pkgs/desktops/gnome-3/update.nix b/pkgs/desktops/gnome-3/update.nix index b7a6ce16d11f..216ae9cb8b38 100644 --- a/pkgs/desktops/gnome-3/update.nix +++ b/pkgs/desktops/gnome-3/update.nix @@ -1,9 +1,10 @@ -{ lib, writeScript, python3, common-updater-scripts, coreutils, gnugrep, gnused }: +{ stdenv, lib, writeScript, python3, common-updater-scripts, coreutils, gnugrep, gnused }: { packageName, attrPath ? packageName, versionPolicy ? "odd-unstable" }: let python = python3.withPackages (p: [ p.requests ]); in writeScript "update-${packageName}" '' + #!${stdenv.shell} set -o errexit PATH=${lib.makeBinPath [ common-updater-scripts coreutils gnugrep gnused python ]} latest_tag=$(python "${./find-latest-version.py}" "${packageName}" "${versionPolicy}" "stable") diff --git a/pkgs/development/libraries/safefile/default.nix b/pkgs/development/libraries/safefile/default.nix index 159bad4c68ea..d09e45a2d1e9 100644 --- a/pkgs/development/libraries/safefile/default.nix +++ b/pkgs/development/libraries/safefile/default.nix @@ -13,6 +13,7 @@ stdenv.mkDerivation rec { passthru = { updateScript = '' + #!${stdenv.shell} cd ${toString ./.} ${toString path}/pkgs/build-support/upstream-updater/update-walker.sh default.nix ''; diff --git a/pkgs/development/tools/build-managers/gup/default.nix b/pkgs/development/tools/build-managers/gup/default.nix index f6d5b7b9b59c..7fd801488311 100644 --- a/pkgs/development/tools/build-managers/gup/default.nix +++ b/pkgs/development/tools/build-managers/gup/default.nix @@ -16,6 +16,7 @@ stdenv.mkDerivation rec { cp -r python/bin $out/bin ''; passthru.updateScript = '' + #!${stdenv.shell} set -e echo cd ${toString ./.} diff --git a/pkgs/development/web/nodejs/update.nix b/pkgs/development/web/nodejs/update.nix index 9ff11982b656..bf6951dc6882 100644 --- a/pkgs/development/web/nodejs/update.nix +++ b/pkgs/development/web/nodejs/update.nix @@ -1,4 +1,5 @@ -{ lib +{ stdenv +, lib , writeScript , coreutils , curl @@ -11,6 +12,7 @@ }: writeScript "update-nodejs" '' + #!${stdenv.shell} PATH=${lib.makeBinPath [ common-updater-scripts coreutils curl gnugrep jq gnupg nix ]} HOME=`mktemp -d` diff --git a/pkgs/os-specific/linux/tp_smapi/default.nix b/pkgs/os-specific/linux/tp_smapi/default.nix index 25eeb889cc28..89c6639748d0 100644 --- a/pkgs/os-specific/linux/tp_smapi/default.nix +++ b/pkgs/os-specific/linux/tp_smapi/default.nix @@ -34,7 +34,7 @@ stdenv.mkDerivation rec { enableParallelBuilding = true; passthru.updateScript = import ./update.nix { - inherit lib writeScript coreutils gnugrep jq curl common-updater-scripts; + inherit stdenv lib writeScript coreutils gnugrep jq curl common-updater-scripts; }; meta = { diff --git a/pkgs/os-specific/linux/tp_smapi/update.nix b/pkgs/os-specific/linux/tp_smapi/update.nix index 94eb44b744cb..1b6dfd90b1e4 100644 --- a/pkgs/os-specific/linux/tp_smapi/update.nix +++ b/pkgs/os-specific/linux/tp_smapi/update.nix @@ -1,6 +1,7 @@ -{ lib, writeScript, coreutils, curl, gnugrep, jq, common-updater-scripts }: +{ stdenv, lib, writeScript, coreutils, curl, gnugrep, jq, common-updater-scripts }: writeScript "update-tp_smapi" '' +#!${stdenv.shell} PATH=${lib.makeBinPath [ common-updater-scripts coreutils curl gnugrep jq ]} tags=`curl -s https://api.github.com/repos/evgeni/tp_smapi/tags` diff --git a/pkgs/tools/package-management/nix-pin/default.nix b/pkgs/tools/package-management/nix-pin/default.nix index f5d62b250e22..063c173a4019 100644 --- a/pkgs/tools/package-management/nix-pin/default.nix +++ b/pkgs/tools/package-management/nix-pin/default.nix @@ -26,6 +26,7 @@ let self = stdenv.mkDerivation rec { let impl = import "${self}/share/nix/api.nix" { inherit pkgs pinConfig; }; in { inherit (impl) augmentedPkgs pins callPackage; }; updateScript = '' + #!${stdenv.shell} set -e echo cd ${toString ./.} diff --git a/pkgs/tools/package-management/nix-update-source/default.nix b/pkgs/tools/package-management/nix-update-source/default.nix index e7eb497b4b9f..7584496f2585 100644 --- a/pkgs/tools/package-management/nix-update-source/default.nix +++ b/pkgs/tools/package-management/nix-update-source/default.nix @@ -1,4 +1,4 @@ -{ lib, pkgs, fetchFromGitHub, python3Packages, nix-prefetch-scripts }: +{ stdenv, lib, pkgs, fetchFromGitHub, python3Packages, nix-prefetch-scripts }: python3Packages.buildPythonApplication rec { version = "0.6.3"; name = "nix-update-source-${version}"; @@ -28,6 +28,7 @@ python3Packages.buildPythonApplication rec { overrideSrc = drv: lib.overrideDerivation drv (orig: { inherit src; }); }; updateScript = '' + #!${stdenv.shell} set -e echo cd ${toString ./.}