yarn2nix: init at 0.1.0 (#35340)

pkgs/development/tools/yarn2nix/bin/yarn2nix.js

@@ -0,0 +1,144 @@
+#!/usr/bin/env node
+"use strict";
+
+const crypto = require('crypto');
+const fs = require("fs");
+const https = require("https");
+const path = require("path");
+const util = require("util");
+
+const lockfile = require("@yarnpkg/lockfile")
+const docopt = require("docopt").docopt;
+
+////////////////////////////////////////////////////////////////////////////////
+
+const USAGE = `
+Usage: yarn2nix [options]
+
+Options:
+  -h --help        Shows this help.
+  --no-nix         Hide the nix output
+  --no-patch       Don't patch the lockfile if hashes are missing
+  --lockfile=FILE  Specify path to the lockfile [default: ./yarn.lock].
+`
+
+const HEAD = `
+{fetchurl, linkFarm}: rec {
+  offline_cache = linkFarm "offline" packages;
+  packages = [
+`.trim();
+
+////////////////////////////////////////////////////////////////////////////////
+
+function generateNix(lockedDependencies) {
+  let found = {};
+
+  console.log(HEAD)
+
+  for (var depRange in lockedDependencies) {
+    let dep = lockedDependencies[depRange];
+
+    let depRangeParts = depRange.split('@');
+    let [url, sha1] = dep["resolved"].split("#");
+    let file_name = path.basename(url)
+
+    if (found.hasOwnProperty(file_name)) {
+      continue;
+    } else {
+      found[file_name] = null;
+    }
+
+
+    console.log(`
+    {
+      name = "${file_name}";
+      path = fetchurl {
+        name = "${file_name}";
+        url  = "${url}";
+        sha1 = "${sha1}";
+      };
+    }`)
+  }
+
+  console.log("  ];")
+  console.log("}")
+}
+
+
+function getSha1(url) {
+  return new Promise((resolve, reject) => {
+    https.get(url, (res) => {
+      const { statusCode } = res;
+      const hash = crypto.createHash('sha1');
+      if (statusCode !== 200) {
+        const err = new Error('Request Failed.\n' +
+                          `Status Code: ${statusCode}`);
+        // consume response data to free up memory
+        res.resume();
+        reject(err);
+      }
+
+      res.on('data', (chunk) => { hash.update(chunk); });
+      res.on('end', () => { resolve(hash.digest('hex')) });
+      res.on('error', reject);
+    });
+  });
+};
+
+function updateResolvedSha1(pkg) {
+  // local dependency
+  if (!pkg.resolved) { return Promise.resolve(); }
+  let [url, sha1] = pkg.resolved.split("#", 2)
+  if (!sha1) {
+    return new Promise((resolve, reject) => {
+      getSha1(url).then(sha1 => {
+        pkg.resolved = `${url}#${sha1}`;
+        resolve();
+      }).catch(reject);
+    });
+  } else {
+    // nothing to do
+    return Promise.resolve();
+  };
+}
+
+function values(obj) {
+  var entries = [];
+  for (let key in obj) {
+    entries.push(obj[key]);
+  }
+  return entries;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Main
+////////////////////////////////////////////////////////////////////////////////
+
+var options = docopt(USAGE);
+
+let data = fs.readFileSync(options['--lockfile'], 'utf8')
+let json = lockfile.parse(data)
+if (json.type != "success") {
+  throw new Error("yarn.lock parse error")
+}
+
+// Check fore missing hashes in the yarn.lock and patch if necessary
+var pkgs = values(json.object);
+Promise.all(pkgs.map(updateResolvedSha1)).then(() => {
+  let newData = lockfile.stringify(json.object);
+
+  if (newData != data) {
+    console.error("found changes in the lockfile", options["--lockfile"]);
+
+    if (options["--no-patch"]) {
+      console.error("...aborting");
+      process.exit(1);
+    }
+
+    fs.writeFileSync(options['--lockfile'], newData);
+  }
+
+  if (!options['--no-nix']) {
+    generateNix(json.object);
+  }
+})

pkgs/development/tools/yarn2nix/default.nix

@@ -0,0 +1,199 @@
+{ stdenv, lib, fetchurl, linkFarm, runCommand, nodejs, yarn }:
+
+let
+  unlessNull = item: alt:
+    if item == null then alt else item;
+
+  yarn2nix = mkYarnPackage {
+    src = ./.;
+    yarnNix = ./yarn.nix;
+
+    passthru = {
+      inherit
+        defaultYarnFlags
+        linkNodeModulesHook
+        mkYarnModules
+        mkYarnNix
+        mkYarnPackage
+        # Export yarn again to make it easier to find out which yarn was used.
+        yarn
+        ;
+    };
+
+    meta = with lib; {
+      description = "generate nix expressions from a yarn.lock file";
+      homepage = "https://github.com/moretea/yarn2nix";
+      license = licenses.gpl3;
+      maintainers = with maintainers; [ manveru zimbatm ];
+    };
+  };
+
+  # Generates the yarn.nix from the yarn.lock file
+  mkYarnNix = yarnLock:
+    runCommand "yarn.nix" {}
+      "${yarn2nix}/bin/yarn2nix --lockfile ${yarnLock} --no-patch > $out";
+
+  # Loads the generated offline cache. This will be used by yarn as
+  # the package source.
+  importOfflineCache = yarnNix:
+    let
+      pkg = import yarnNix { inherit fetchurl linkFarm; };
+    in
+      pkg.offline_cache;
+
+  defaultYarnFlags = [
+    "--offline"
+    "--frozen-lockfile"
+    "--ignore-engines"
+    "--ignore-scripts"
+  ];
+
+  mkYarnModules = {
+    name,
+    packageJSON,
+    yarnLock,
+    yarnNix ? mkYarnNix yarnLock,
+    yarnFlags ? defaultYarnFlags,
+    pkgConfig ? {},
+    preBuild ? "",
+  }:
+    let
+      offlineCache = importOfflineCache yarnNix;
+      extraBuildInputs = (lib.flatten (builtins.map (key:
+        pkgConfig.${key} . buildInputs or []
+      ) (builtins.attrNames pkgConfig)));
+      postInstall = (builtins.map (key:
+        if (pkgConfig.${key} ? postInstall) then
+          ''
+            for f in $(find -L -path '*/node_modules/${key}' -type d); do
+              (cd "$f" && (${pkgConfig.${key}.postInstall}))
+            done
+          ''
+        else
+          ""
+      ) (builtins.attrNames pkgConfig));
+    in
+    stdenv.mkDerivation {
+      inherit name preBuild;
+      phases = ["configurePhase" "buildPhase"];
+      buildInputs = [ yarn nodejs ] ++ extraBuildInputs;
+
+      configurePhase = ''
+        # Yarn writes cache directories etc to $HOME.
+        export HOME=$PWD/yarn_home
+      '';
+
+      buildPhase = ''
+        runHook preBuild
+
+        cp ${packageJSON} ./package.json
+        cp ${yarnLock} ./yarn.lock
+        chmod +w ./yarn.lock
+
+        yarn config --offline set yarn-offline-mirror ${offlineCache}
+
+        # Do not look up in the registry, but in the offline cache.
+        # TODO: Ask upstream to fix this mess.
+        sed -i -E 's|^(\s*resolved\s*")https?://.*/|\1|' yarn.lock
+        yarn install ${lib.escapeShellArgs yarnFlags}
+
+        ${lib.concatStringsSep "\n" postInstall}
+
+        mkdir $out
+        mv node_modules $out/
+        patchShebangs $out
+      '';
+    };
+
+  # This can be used as a shellHook in mkYarnPackage. It brings the built node_modules into
+  # the shell-hook environment.
+  linkNodeModulesHook = ''
+    if [[ -d node_modules || -L node_modules ]]; then
+      echo "./node_modules is present. Replacing."
+      rm -rf node_modules
+    fi
+
+    ln -s "$node_modules" node_modules
+  '';
+
+  mkYarnPackage = {
+    name ? null,
+    src,
+    packageJSON ? src + "/package.json",
+    yarnLock ? src + "/yarn.lock",
+    yarnNix ? mkYarnNix yarnLock,
+    yarnFlags ? defaultYarnFlags,
+    yarnPreBuild ? "",
+    pkgConfig ? {},
+    extraBuildInputs ? [],
+    publishBinsFor ? null,
+    ...
+  }@attrs:
+    let
+      package = lib.importJSON packageJSON;
+      pname = package.name;
+      version = package.version;
+      deps = mkYarnModules {
+        name = "${pname}-modules-${version}";
+        preBuild = yarnPreBuild;
+        inherit packageJSON yarnLock yarnNix yarnFlags pkgConfig;
+      };
+      publishBinsFor_ = unlessNull publishBinsFor [pname];
+    in stdenv.mkDerivation (builtins.removeAttrs attrs ["pkgConfig"] // {
+      inherit src;
+
+      name = unlessNull name "${pname}-${version}";
+
+      buildInputs = [ yarn nodejs ] ++ extraBuildInputs;
+
+      node_modules = deps + "/node_modules";
+
+      configurePhase = attrs.configurePhase or ''
+        runHook preConfigure
+
+        if [ -d npm-packages-offline-cache ]; then
+          echo "npm-pacakges-offline-cache dir present. Removing."
+          rm -rf npm-packages-offline-cache
+        fi
+
+        if [[ -d node_modules || -L node_modules ]]; then
+          echo "./node_modules is present. Removing."
+          rm -rf node_modules
+        fi
+
+        mkdir -p node_modules
+        ln -s $node_modules/* node_modules/
+        ln -s $node_modules/.bin node_modules/
+
+        if [ -d node_modules/${pname} ]; then
+          echo "Error! There is already an ${pname} package in the top level node_modules dir!"
+          exit 1
+        fi
+
+        runHook postConfigure
+      '';
+
+      # Replace this phase on frontend packages where only the generated
+      # files are an interesting output.
+      installPhase = attrs.installPhase or ''
+        runHook preInstall
+
+        mkdir -p $out
+        cp -r node_modules $out/node_modules
+        cp -r . $out/node_modules/${pname}
+        rm -rf $out/node_modules/${pname}/node_modules
+
+        mkdir $out/bin
+        node ${./fixup_bin.js} $out ${lib.concatStringsSep " " publishBinsFor_}
+
+        runHook postInstall
+      '';
+
+      passthru = {
+        inherit package deps;
+      } // (attrs.passthru or {});
+
+      # TODO: populate meta automatically
+    });
+in
+  yarn2nix

pkgs/development/tools/yarn2nix/fixup_bin.js

@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+"use strict";
+
+/* Usage:
+ * node fixup_bin.js <output_dir> [<bin_pkg_1>, <bin_pkg_2> ... ]
+ */
+
+const fs = require("fs");
+const path = require("path");
+
+const output = process.argv[2];
+const packages_to_publish_bin = process.argv.slice(3);
+const derivation_bin_path = output + "/bin";
+
+function processPackage(name) {
+  console.log("Processing ", name);
+  const package_path = output + "/node_modules/" + name;
+  const package_json_path = package_path + "/package.json";
+  const package_json = JSON.parse(fs.readFileSync(package_json_path));
+
+  if (!package_json.bin) {
+    console.log("No binaries provided");
+    return;
+  }
+
+  // There are two alternative syntaxes for `bin`
+  // a) just a plain string, in which case the name of the package is the name of the binary.
+  // b) an object, where key is the name of the eventual binary, and the value the path to that binary.
+  if (typeof package_json.bin == "string") {
+    let bin_name = package_json.bin;
+    package_json.bin = { };
+    package_json.bin[package_json.name] = bin_name;
+  }
+
+  for (let binName in package_json.bin) {
+    const bin_path = package_json.bin[binName];
+    const full_bin_path = path.normalize(package_path + "/" + bin_path);
+    fs.symlinkSync(full_bin_path, derivation_bin_path + "/"+ binName);
+    console.log("Linked", binName);
+  }
+}
+
+packages_to_publish_bin.forEach((pkg) => {
+  processPackage(pkg);
+});

pkgs/development/tools/yarn2nix/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "yarn2nix",
+  "version": "1.0.0",
+  "description": "Convert packages.json and yarn.lock into a Nix expression that downloads all the dependencies",
+  "main": "index.js",
+  "repository": ".",
+  "author": "Maarten Hoogendoorn <maarten@moretea.nl>",
+  "license": "MIT",
+  "scripts": {
+    "yarn2nix": "bin/yarn2nix.js"
+  },
+  "bin": {
+    "yarn2nix": "bin/yarn2nix.js"
+  },
+  "dependencies": {
+    "@yarnpkg/lockfile": "^1.0.0",
+    "docopt": "^0.6.2"
+  }
+}

pkgs/development/tools/yarn2nix/yarn.lock

@@ -0,0 +1,11 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"@yarnpkg/lockfile@^1.0.0":
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/@yarnpkg/lockfile/-/lockfile-1.0.0.tgz#33d1dbb659a23b81f87f048762b35a446172add3"
+
+docopt@^0.6.2:
+  version "0.6.2"
+  resolved "https://registry.yarnpkg.com/docopt/-/docopt-0.6.2.tgz#b28e9e2220da5ec49f7ea5bb24a47787405eeb11"

pkgs/development/tools/yarn2nix/yarn.nix

@@ -0,0 +1,23 @@
+{fetchurl, linkFarm}: rec {
+  offline_cache = linkFarm "offline" packages;
+  packages = [
+
+    {
+      name = "lockfile-1.0.0.tgz";
+      path = fetchurl {
+        name = "lockfile-1.0.0.tgz";
+        url  = "https://registry.yarnpkg.com/@yarnpkg/lockfile/-/lockfile-1.0.0.tgz";
+        sha1 = "33d1dbb659a23b81f87f048762b35a446172add3";
+      };
+    }
+
+    {
+      name = "docopt-0.6.2.tgz";
+      path = fetchurl {
+        name = "docopt-0.6.2.tgz";
+        url  = "https://registry.yarnpkg.com/docopt/-/docopt-0.6.2.tgz";
+        sha1 = "b28e9e2220da5ec49f7ea5bb24a47787405eeb11";
+      };
+    }
+  ];
+}

pkgs/top-level/all-packages.nix

@@ -5603,6 +5603,9 @@ with pkgs;
 
   yarn = callPackage ../development/tools/yarn  { };
 
+  yarn2nix = callPackage ../development/tools/yarn2nix { };
+  inherit (yarn2nix) mkYarnPackage;
+
   yasr = callPackage ../applications/audio/yasr { };
 
   yank = callPackage ../tools/misc/yank { };