1
0
Fork 1
mirror of https://github.com/NixOS/nixpkgs.git synced 2024-12-25 03:17:13 +00:00
nixpkgs/pkgs/build-support/rust/fetch-cargo-deps
Benno Fünfstück 19d3cf81d3 rust: fix fetch-cargo-deps for git dependencies
We need to make sure that `$revs` ends with a space, since files must always
end with newlines. The previous code ignored the last entry in `$revs`, because
read already returns non-zero exit code for the last entry, as it does not end
with a space.
2017-05-23 10:01:33 +02:00

210 lines
7.1 KiB
Plaintext
Executable file

# copied from libgit2 source code 'repo-template.h'
makeGitTemplate() {
local target="$1"
mkdir -p -m777 "$target/info" "$target/pack" "$target/objects" "$target/refs"
mkdir -p -m777 "$target/refs/heads" "$target/refs/tags" "$target/objects/info" "$target/objects/pack"
cat <<'EOF' > "$target/description"
Unnamed repository; edit this file 'description' to name the repository.
EOF
chmod 666 "$target/description"
cat <<'EOF' > "$target/info/exclude"
# File patterns to ignore; see `git help ignore` for more information.
# Lines that start with '#' are comments.
EOF
}
fetchCargoDeps() {
src=$(realpath $1)
out=$(realpath $2)
echo "Fetching $src to $out"
mkdir $out
# Configure git template dir to make libgit2 more deterministic
#
# Without a template dir, libgit2 defaults to /usr/share/git-core/templates,
# which can vary between systems if sandboxed builds aren't used.
#
# Note: we explictly set --tmpdir for mktemp here to make it more friendly
# for nix-shell users, where $TMPDIR is not necessarily set to NIX_BUILD_TOP
echo "Setting up git templatedir"
export GIT_TEMPLATE_DIR="$(mktemp -d --tmpdir=$NIX_BUILD_TOP git-template.XXX)"
makeGitTemplate "$GIT_TEMPLATE_DIR"
export XDG_CONFIG_HOME="$(mktemp -d --tmpdir=$NIX_BUILD_TOP home.XXX)"
mkdir -p $XDG_CONFIG_HOME/git
cat <<EOF > $XDG_CONFIG_HOME/git/config
[init]
templatedir = $GIT_TEMPLATE_DIR
EOF
# Configure cargo to fetch from a local copy of the crates.io registry
echo "Using rust registry from $rustRegistry"
cat <<EOF > $out/config
[registry]
index = "file://$rustRegistry"
EOF
export CARGO_HOME=$out
cd $src
if [[ ! -f Cargo.lock ]]; then
echo
echo "ERROR: The Cargo.lock file doesn't exist"
echo
echo "Cargo.lock is needed to make sure that depsSha256 doesn't change"
echo "when the registry is updated."
echo
exit 1
fi
# We need to do the following string replacement so that 'cargo fetch'
# doesn't ignore the versions specified in Cargo.lock
substituteInPlace Cargo.lock \
--replace "registry+https://github.com/rust-lang/crates.io-index" \
"registry+file://$rustRegistry"
# Do any possible 'cargo update -p <pkgName> --precise <version>' ad-hoc updates
eval "$cargoUpdateHook"
# Do the fetch
cargo fetch --verbose
# Now that we have fetched everything, let's make the output deterministic
# Cargo uses the following directory structure for fetched data, where
# $indexHash is a hash of the registry index URL:
#
#
# /config:
#
# Cargo config file. We'll delete this because it's not deterministic,
# and instead recreate it just before running 'cargo build'.
#
# /registry/cache/$indexHash/:
#
# This is where tarballs of registry package dependencies are kept
# We'll need to keep this, but make sure $indexHash is a fixed name.
#
# /registry/index/$indexHash/:
#
# A copy of the registry index is kept here. We can delete this, and
# instead, just before running 'cargo build', we'll symlink this
# directory to our static copy of the registry in the Nix store.
#
# /registry/src/$indexHash/{pkgName-pkgVersion}/:
#
# Here cargo keeps extracted sources of the cached tarballs.
# We'll just delete this because cargo will re-populate them from the
# tarballs.
#
# /git/db/{domain-hash}/:
#
# Here cargo keeps the `.git` directories of git dependencies.
# We'll need to keep these, but make them deterministic.
#
# /git/checkouts/{domain-hash}/{branchName}/:
#
# Here cargo keeps checked-out sources of the git dependencies.
# We can delete this, because cargo will re-populate them from the above
# `.git` directories.
#
# Let's start
# Remove cargo config file, which points to the ever-changing registry
rm $out/config
# Save the Cargo.lock file into the output, so that we don't have to do another
# 'cargo update' during the build (which would try to access the network) for
# any ad-hoc package updates (through $cargoUpdateHook).
#
# We need to replace the rustRegistry URL with something deterministic.
# Since the URL won't actually be accessed anymore, it's fine to use /dev/null.
substituteInPlace Cargo.lock \
--replace "registry+file://$rustRegistry" \
"registry+file:///dev/null"
mv Cargo.lock $out/
# Let's replace $indexHash with something more deterministic
mv $out/registry/cache/* $out/registry/cache/HASH
# The registry index changes all the time, so it's not deterministic
# We'll symlink it before running 'cargo build'
rm -rf $out/registry/index/*
# Make git DBs deterministic
# TODO: test with git submodules
[[ ! -d $out/git/checkouts ]] || (cd $out/git/checkouts && for name in *; do
revs=""
cd "$out/git/checkouts/$name"
while read dir; do
# extract substring: [dir = "./xxx/yyy/.git"] => [branch = "xxx/yyy"]
branch="${dir:2:$((${#dir}-7))}"
cd "$out/git/checkouts/$name/$branch"
rev="$(git rev-parse HEAD)"
revs="$rev $revs"
done < <(find . -type d -name .git -print)
echo "List of revs to keep for git db $name: $revs"
(
# The following code was adapted from nix-prefetch-git
cd "$out/git/db/$name"
export GIT_DIR=.
# Remove all remote branches
git branch -r | while read branch; do
git branch -rD "$branch" >&2
done
# Remove all tags
git tag | while read tag; do
git tag -d "$tag" >&2
done
# Remove all local branches
branchrefs=()
eval "$(git for-each-ref --shell --format='branchrefs+=(%(refname))' refs/heads/)"
for branchref in "${branchrefs[@]}"; do
git update-ref -d "$branchref" >&2
done
# Create ad-hoc branches for the revs we need
echo "$revs" | while read -d " " rev; do
echo "Creating git branch b_$rev $rev"
git branch b_$rev $rev
done
# Remove files that have timestamps or otherwise have non-deterministic
# properties.
rm -rf logs/ hooks/ index FETCH_HEAD ORIG_HEAD refs/remotes/origin/HEAD config
# Do a full repack. Must run single-threaded, or else we lose determinism.
git config pack.threads 1
git repack -A -d -f
rm -f config
# Garbage collect unreferenced objects.
git gc --prune=all
)
done)
# Remove unneeded outputs
[[ ! -d $out/registry/src ]] || rm -rf $out/registry/src
[[ ! -d $out/git/checkouts ]] || rm -rf $out/git/checkouts
# XXX: provide some debugging output to see find out why we are seeing
# sporadic hash mismatches
find $out ! -type f
find $out -type f -exec sha256sum {} +
}