diff --git a/doc/builders/fetchers.chapter.md b/doc/builders/fetchers.chapter.md index 5b28b2dcb398..28388ba685d8 100644 --- a/doc/builders/fetchers.chapter.md +++ b/doc/builders/fetchers.chapter.md @@ -40,6 +40,24 @@ Used with Git. Expects `url` to a Git repo, `rev`, and `sha256`. `rev` in this c Additionally the following optional arguments can be given: `fetchSubmodules = true` makes `fetchgit` also fetch the submodules of a repository. If `deepClone` is set to true, the entire repository is cloned as opposing to just creating a shallow clone. `deepClone = true` also implies `leaveDotGit = true` which means that the `.git` directory of the clone won't be removed after checkout. +If only parts of the repository are needed, `sparseCheckout` can be used. This will prevent git from fetching unnecessary blobs from server, see [git sparse-checkout](https://git-scm.com/docs/git-sparse-checkout) and [git clone --filter](https://git-scm.com/docs/git-clone#Documentation/git-clone.txt---filterltfilter-specgt) for more infomation: + +```nix +{ stdenv, fetchgit }: + +stdenv.mkDerivation { + name = "hello"; + src = fetchgit { + url = "https://..."; + sparseCheckout = '' + path/to/be/included + another/path + ''; + sha256 = "0000000000000000000000000000000000000000000000000000"; + }; +} +``` + ## `fetchfossil` {#fetchfossil} Used with Fossil. Expects `url` to a Fossil archive, `rev`, and `sha256`. diff --git a/pkgs/build-support/fetchgit/builder.sh b/pkgs/build-support/fetchgit/builder.sh index 0047a335c76c..c7c7d21709a1 100644 --- a/pkgs/build-support/fetchgit/builder.sh +++ b/pkgs/build-support/fetchgit/builder.sh @@ -11,6 +11,7 @@ $SHELL $fetcher --builder --url "$url" --out "$out" --rev "$rev" \ ${fetchLFS:+--fetch-lfs} \ ${deepClone:+--deepClone} \ ${fetchSubmodules:+--fetch-submodules} \ + ${sparseCheckout:+--sparse-checkout "$sparseCheckout"} \ ${branchName:+--branch-name "$branchName"} runHook postFetch diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index 3a05008dacb5..1b59668ce4bc 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -15,6 +15,7 @@ in { url, rev ? "HEAD", md5 ? "", sha256 ? "", hash ? "", leaveDotGit ? deepClone , fetchSubmodules ? true, deepClone ? false , branchName ? null +, sparseCheckout ? "" , name ? urlToName url rev , # Shell code executed after the file has been fetched # successfully. This can do things like check or transform the file. @@ -74,7 +75,7 @@ stdenvNoCC.mkDerivation { else lib.fakeSha256; - inherit url rev leaveDotGit fetchLFS fetchSubmodules deepClone branchName postFetch; + inherit url rev leaveDotGit fetchLFS fetchSubmodules deepClone branchName sparseCheckout postFetch; postHook = if netrcPhase == null then null else '' ${netrcPhase} diff --git a/pkgs/build-support/fetchgit/nix-prefetch-git b/pkgs/build-support/fetchgit/nix-prefetch-git index be5068c7ad31..4e6f25b8dd7d 100755 --- a/pkgs/build-support/fetchgit/nix-prefetch-git +++ b/pkgs/build-support/fetchgit/nix-prefetch-git @@ -48,6 +48,7 @@ Options: --rev ref Any sha1 or references (such as refs/heads/master) --hash h Expected hash. --branch-name Branch name to check out into + --sparse-checkout Only fetch and checkout part of the repository. --deepClone Clone the entire repository. --no-deepClone Make a shallow clone of just the required ref. --leave-dotGit Keep the .git directories. @@ -75,6 +76,7 @@ for arg; do --hash) argfun=set_hashType;; --branch-name) argfun=set_branchName;; --deepClone) deepClone=true;; + --sparse-checkout) argfun=set_sparseCheckout;; --quiet) QUIET=true;; --no-deepClone) deepClone=;; --leave-dotGit) leaveDotGit=true;; @@ -96,7 +98,7 @@ for arg; do case $argfun in set_*) var=${argfun#set_} - eval $var=$arg + eval "$var=$(printf %q "$arg")" ;; esac argfun="" @@ -112,6 +114,10 @@ init_remote(){ local url=$1 clean_git init --initial-branch=master clean_git remote add origin "$url" + if [ -n "$sparseCheckout" ]; then + git config remote.origin.partialclonefilter "blob:none" + echo "$sparseCheckout" | git sparse-checkout set --stdin + fi ( [ -n "$http_proxy" ] && clean_git config http.proxy "$http_proxy" ) || true } diff --git a/pkgs/build-support/fetchgit/tests.nix b/pkgs/build-support/fetchgit/tests.nix index 6805473e2968..c558fb6efa4d 100644 --- a/pkgs/build-support/fetchgit/tests.nix +++ b/pkgs/build-support/fetchgit/tests.nix @@ -7,4 +7,15 @@ rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a"; sha256 = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY="; }; + + sparseCheckout = invalidateFetcherByDrvHash fetchgit { + name = "nix-source"; + url = "https://github.com/NixOS/nix"; + rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a"; + sparseCheckout = '' + src + tests + ''; + sha256 = "sha256-FknO6C/PSnMPfhUqObD4vsW4PhkwdmPa9blNzcNvJQ4="; + }; } diff --git a/pkgs/build-support/fetchgithub/default.nix b/pkgs/build-support/fetchgithub/default.nix index d40a0478baba..4e4162368b91 100644 --- a/pkgs/build-support/fetchgithub/default.nix +++ b/pkgs/build-support/fetchgithub/default.nix @@ -3,6 +3,7 @@ { owner, repo, rev, name ? "source" , fetchSubmodules ? false, leaveDotGit ? null , deepClone ? false, private ? false, forceFetchGit ? false +, sparseCheckout ? "" , githubBase ? "github.com", varPrefix ? null , ... # For hash agility }@args: @@ -10,7 +11,7 @@ let baseUrl = "https://${githubBase}/${owner}/${repo}"; passthruAttrs = removeAttrs args [ "owner" "repo" "rev" "fetchSubmodules" "forceFetchGit" "private" "githubBase" "varPrefix" ]; varBase = "NIX${if varPrefix == null then "" else "_${varPrefix}"}_GITHUB_PRIVATE_"; - useFetchGit = fetchSubmodules || (leaveDotGit == true) || deepClone || forceFetchGit; + useFetchGit = fetchSubmodules || (leaveDotGit == true) || deepClone || forceFetchGit || (sparseCheckout != ""); # We prefer fetchzip in cases we don't need submodules as the hash # is more stable in that case. fetcher = if useFetchGit then fetchgit else fetchzip; @@ -30,7 +31,7 @@ let }; fetcherArgs = (if useFetchGit then { - inherit rev deepClone fetchSubmodules; url = "${baseUrl}.git"; + inherit rev deepClone fetchSubmodules sparseCheckout; url = "${baseUrl}.git"; } // lib.optionalAttrs (leaveDotGit != null) { inherit leaveDotGit; } else { url = "${baseUrl}/archive/${rev}.tar.gz"; } ) // privateAttrs // passthruAttrs // { inherit name; };