Merge pull request #53934 from Mic92/prefer-fetch-remote

prefer-fetch-remote: an overlay to fetch on remote builders

doc/functions.xml

@@ -14,4 +14,5 @@
  <xi:include href="functions/fhs-environments.xml" />
  <xi:include href="functions/shell.xml" />
  <xi:include href="functions/dockertools.xml" />
+ <xi:include href="functions/prefer-remote-fetch.xml" />
 </chapter>

doc/functions/prefer-remote-fetch.xml

@@ -0,0 +1,27 @@
+<section xmlns="http://docbook.org/ns/docbook"
+         xmlns:xlink="http://www.w3.org/1999/xlink"
+         xmlns:xi="http://www.w3.org/2001/xinclude"
+         xml:id="sec-prefer-remote-fetch">
+ <title>prefer-remote-fetch overlay</title>
+
+ <para>
+  <function>prefer-remote-fetch</function> is an overlay that download sources
+  on remote builder. This is useful when the evaluating machine has a slow
+  upload while the builder can fetch faster directly from the source.
+  To use it, put the following snippet as a new overlay:
+  <programlisting>
+    self: super:
+      (super.prefer-remote-fetch self super)
+  </programlisting>
+
+  A full configuration example for that sets the overlay up for your own account,
+  could look like this
+
+  <programlisting>
+    $ mkdir ~/.config/nixpkgs/overlays/
+    $ cat &gt; ~/.config/nixpkgs/overlays/prefer-remote-fetch.nix &lt;&lt;EOF
+      self: super: super.prefer-remote-fetch self super
+    EOF
+  </programlisting>
+ </para>
+</section>

pkgs/build-support/fetchgit/default.nix

@@ -19,6 +19,7 @@ in
 , # Shell code executed after the file has been fetched
   # successfully. This can do things like check or transform the file.
   postFetch ? ""
+, preferLocalBuild ? true
 }:
 
 /* NOTE:
@@ -66,5 +67,5 @@ stdenvNoCC.mkDerivation {
     "GIT_PROXY_COMMAND" "SOCKS_SERVER"
   ];
 
-  preferLocalBuild = true;
+  inherit preferLocalBuild;
 }

pkgs/build-support/fetchhg/default.nix

@@ -1,4 +1,11 @@
-{stdenvNoCC, mercurial}: {name ? null, url, rev ? null, md5 ? null, sha256 ? null, fetchSubrepos ? false}:
+{ stdenvNoCC, mercurial }:
+{ name ? null
+, url
+, rev ? null
+, md5 ? null
+, sha256 ? null
+, fetchSubrepos ? false
+, preferLocalBuild ? true }:
 
 if md5 != null then
   throw "fetchhg does not support md5 anymore, please use sha256"
@@ -18,5 +25,5 @@ stdenvNoCC.mkDerivation {
   outputHash = sha256;
 
   inherit url rev;
-  preferLocalBuild = true;
+  inherit preferLocalBuild;
 }

pkgs/build-support/fetchipfs/default.nix

@@ -14,6 +14,7 @@
 , meta           ? {}
 , port           ? "8080"
 , postFetch      ? ""
+, preferLocalBuild ? true
 }:
 
 assert sha512 != "" -> builtins.compareVersions "1.11" builtins.nixVersion <= 0;
@@ -42,11 +43,10 @@ if (!hasHash) then throw "Specify sha for fetchipfs fixed-output derivation" els
           postFetch
           ipfs
           url
-          port;
+          port
+          meta;
 
   # Doing the download on a remote machine just duplicates network
   # traffic, so don't do that.
-  preferLocalBuild = true;
-
-  inherit meta;
+  inherit preferLocalBuild;
 }

pkgs/build-support/fetchsvn/default.nix

@@ -1,6 +1,7 @@
 {stdenvNoCC, subversion, glibcLocales, sshSupport ? false, openssh ? null}:
-{url, rev ? "HEAD", md5 ? "", sha256 ? "",
- ignoreExternals ? false, ignoreKeywords ? false, name ? null}:
+{url, rev ? "HEAD", md5 ? "", sha256 ? ""
+, ignoreExternals ? false, ignoreKeywords ? false, name ? null
+, preferLocalBuild ? true }:
 
 let
   repoName = with stdenvNoCC.lib;
@@ -40,5 +41,5 @@ stdenvNoCC.mkDerivation {
   inherit url rev sshSupport openssh ignoreExternals ignoreKeywords;
 
   impureEnvVars = stdenvNoCC.lib.fetchers.proxyImpureEnvVars;
-  preferLocalBuild = true;
+  inherit preferLocalBuild;
 }

pkgs/build-support/fetchurl/default.nix

@@ -87,6 +87,9 @@ in
 
   # Passthru information, if any.
 , passthru ? {}
+  # Doing the download on a remote machine just duplicates network
+  # traffic, so don't do that by default
+, preferLocalBuild ? true
 }:
 
 assert sha512 != "" -> builtins.compareVersions "1.11" builtins.nixVersion <= 0;
@@ -135,9 +138,7 @@ stdenvNoCC.mkDerivation {
 
   nixpkgsVersion = lib.trivial.release;
 
-  # Doing the download on a remote machine just duplicates network
-  # traffic, so don't do that.
-  preferLocalBuild = true;
+  inherit preferLocalBuild;
 
   postHook = if netrcPhase == null then null else ''
     ${netrcPhase}

pkgs/build-support/prefer-remote-fetch/default.nix

@@ -0,0 +1,19 @@
+# An overlay that download sources on remote builder.
+# This is useful when the evaluating machine has a slow
+# upload while the builder can fetch faster directly from the source.
+# Usage: Put the following snippet in your usual overlay definition:
+#
+#   self: super:
+#     (super.prefer-remote-fetch self super)
+# Full configuration example for your own account:
+#
+# $ mkdir ~/.config/nixpkgs/overlays/
+# $ echo 'self: super: super.prefer-remote-fetch self super' > ~/.config/nixpkgs/overlays/prefer-remote-fetch.nix
+#
+self: super: {
+  fetchurl = args: super.fetchurl (args // { preferLocalBuild = false; });
+  fetchgit = args: super.fetchgit (args // { preferLocalBuild = false; });
+  fetchhg = args: super.fetchhg (args // { preferLocalBuild = false; });
+  fetchsvn = args: super.fetchsvn (args // { preferLocalBuild = false; });
+  fetchipfs = args: super.fetchipfs (args // { preferLocalBuild = false; });
+}

pkgs/top-level/all-packages.nix

@@ -208,6 +208,8 @@ in
 
   fetchMavenArtifact = callPackage ../build-support/fetchmavenartifact { };
 
+  prefer-remote-fetch = import ../build-support/prefer-remote-fetch;
+
   global-platform-pro = callPackage ../development/tools/global-platform-pro/default.nix { };
 
   graph-easy = callPackage ../tools/graphics/graph-easy { };