Merge pull request #135881 from azuwis/master

fetchgit: Support sparse checkout

doc/builders/fetchers.chapter.md

@@ -40,6 +40,24 @@ Used with Git. Expects `url` to a Git repo, `rev`, and `sha256`. `rev` in this c
 
 Additionally the following optional arguments can be given: `fetchSubmodules = true` makes `fetchgit` also fetch the submodules of a repository. If `deepClone` is set to true, the entire repository is cloned as opposing to just creating a shallow clone. `deepClone = true` also implies `leaveDotGit = true` which means that the `.git` directory of the clone won't be removed after checkout.
 
+If only parts of the repository are needed, `sparseCheckout` can be used. This will prevent git from fetching unnecessary blobs from server, see [git sparse-checkout](https://git-scm.com/docs/git-sparse-checkout) and [git clone --filter](https://git-scm.com/docs/git-clone#Documentation/git-clone.txt---filterltfilter-specgt) for more infomation:
+
+```nix
+{ stdenv, fetchgit }:
+
+stdenv.mkDerivation {
+  name = "hello";
+  src = fetchgit {
+    url = "https://...";
+    sparseCheckout = ''
+      path/to/be/included
+      another/path
+    '';
+    sha256 = "0000000000000000000000000000000000000000000000000000";
+  };
+}
+```
+
 ## `fetchfossil` {#fetchfossil}
 
 Used with Fossil. Expects `url` to a Fossil archive, `rev`, and `sha256`.

pkgs/build-support/fetchgit/builder.sh

@@ -11,6 +11,7 @@ $SHELL $fetcher --builder --url "$url" --out "$out" --rev "$rev" \
   ${fetchLFS:+--fetch-lfs} \
   ${deepClone:+--deepClone} \
   ${fetchSubmodules:+--fetch-submodules} \
+  ${sparseCheckout:+--sparse-checkout "$sparseCheckout"} \
   ${branchName:+--branch-name "$branchName"}
 
 runHook postFetch

pkgs/build-support/fetchgit/default.nix

@@ -15,6 +15,7 @@ in
 { url, rev ? "HEAD", md5 ? "", sha256 ? "", hash ? "", leaveDotGit ? deepClone
 , fetchSubmodules ? true, deepClone ? false
 , branchName ? null
+, sparseCheckout ? ""
 , name ? urlToName url rev
 , # Shell code executed after the file has been fetched
   # successfully. This can do things like check or transform the file.
@@ -74,7 +75,7 @@ stdenvNoCC.mkDerivation {
   else
     lib.fakeSha256;
 
-  inherit url rev leaveDotGit fetchLFS fetchSubmodules deepClone branchName postFetch;
+  inherit url rev leaveDotGit fetchLFS fetchSubmodules deepClone branchName sparseCheckout postFetch;
 
   postHook = if netrcPhase == null then null else ''
     ${netrcPhase}

pkgs/build-support/fetchgit/nix-prefetch-git

@@ -48,6 +48,7 @@ Options:
       --rev ref       Any sha1 or references (such as refs/heads/master)
       --hash h        Expected hash.
       --branch-name   Branch name to check out into
+      --sparse-checkout Only fetch and checkout part of the repository.
       --deepClone     Clone the entire repository.
       --no-deepClone  Make a shallow clone of just the required ref.
       --leave-dotGit  Keep the .git directories.
@@ -75,6 +76,7 @@ for arg; do
             --hash) argfun=set_hashType;;
             --branch-name) argfun=set_branchName;;
             --deepClone) deepClone=true;;
+            --sparse-checkout) argfun=set_sparseCheckout;;
             --quiet) QUIET=true;;
             --no-deepClone) deepClone=;;
             --leave-dotGit) leaveDotGit=true;;
@@ -96,7 +98,7 @@ for arg; do
         case $argfun in
             set_*)
                 var=${argfun#set_}
-                eval $var=$arg
+                eval "$var=$(printf %q "$arg")"
                 ;;
         esac
         argfun=""
@@ -112,6 +114,10 @@ init_remote(){
     local url=$1
     clean_git init --initial-branch=master
     clean_git remote add origin "$url"
+    if [ -n "$sparseCheckout" ]; then
+        git config remote.origin.partialclonefilter "blob:none"
+        echo "$sparseCheckout" | git sparse-checkout set --stdin
+    fi
     ( [ -n "$http_proxy" ] && clean_git config http.proxy "$http_proxy" ) || true
 }
 

pkgs/build-support/fetchgit/tests.nix

@@ -7,4 +7,15 @@
     rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a";
     sha256 = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY=";
   };
+
+  sparseCheckout = invalidateFetcherByDrvHash fetchgit {
+    name = "nix-source";
+    url = "https://github.com/NixOS/nix";
+    rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a";
+    sparseCheckout = ''
+      src
+      tests
+    '';
+    sha256 = "sha256-FknO6C/PSnMPfhUqObD4vsW4PhkwdmPa9blNzcNvJQ4=";
+  };
 }

pkgs/build-support/fetchgithub/default.nix

@@ -3,6 +3,7 @@
 { owner, repo, rev, name ? "source"
 , fetchSubmodules ? false, leaveDotGit ? null
 , deepClone ? false, private ? false, forceFetchGit ? false
+, sparseCheckout ? ""
 , githubBase ? "github.com", varPrefix ? null
 , ... # For hash agility
 }@args:
@@ -10,7 +11,7 @@ let
   baseUrl = "https://${githubBase}/${owner}/${repo}";
   passthruAttrs = removeAttrs args [ "owner" "repo" "rev" "fetchSubmodules" "forceFetchGit" "private" "githubBase" "varPrefix" ];
   varBase = "NIX${if varPrefix == null then "" else "_${varPrefix}"}_GITHUB_PRIVATE_";
-  useFetchGit = fetchSubmodules || (leaveDotGit == true) || deepClone || forceFetchGit;
+  useFetchGit = fetchSubmodules || (leaveDotGit == true) || deepClone || forceFetchGit || (sparseCheckout != "");
   # We prefer fetchzip in cases we don't need submodules as the hash
   # is more stable in that case.
   fetcher = if useFetchGit then fetchgit else fetchzip;
@@ -30,7 +31,7 @@ let
   };
   fetcherArgs = (if useFetchGit
     then {
-      inherit rev deepClone fetchSubmodules; url = "${baseUrl}.git";
+      inherit rev deepClone fetchSubmodules sparseCheckout; url = "${baseUrl}.git";
     } // lib.optionalAttrs (leaveDotGit != null) { inherit leaveDotGit; }
     else { url = "${baseUrl}/archive/${rev}.tar.gz"; }
   ) // privateAttrs // passthruAttrs // { inherit name; };