2019-07-25 10:43:45 +01:00
|
|
|
{ stdenv
|
|
|
|
, bazel
|
|
|
|
, cacert
|
|
|
|
, lib
|
|
|
|
}:
|
2017-11-10 17:06:26 +00:00
|
|
|
|
2020-02-25 16:00:44 +00:00
|
|
|
let
|
|
|
|
bazelPkg = bazel;
|
|
|
|
in
|
|
|
|
|
2019-11-24 04:25:02 +00:00
|
|
|
args@{
|
|
|
|
name
|
2020-02-25 16:00:44 +00:00
|
|
|
, bazel ? bazelPkg
|
2019-11-24 04:25:02 +00:00
|
|
|
, bazelFlags ? []
|
|
|
|
, bazelBuildFlags ? []
|
|
|
|
, bazelFetchFlags ? []
|
|
|
|
, bazelTarget
|
|
|
|
, buildAttrs
|
|
|
|
, fetchAttrs
|
|
|
|
|
|
|
|
# Newer versions of Bazel are moving away from built-in rules_cc and instead
|
|
|
|
# allow fetching it as an external dependency in a WORKSPACE file[1]. If
|
|
|
|
# removed in the fixed-output fetch phase, building will fail to download it.
|
|
|
|
# This can be seen e.g. in #73097
|
|
|
|
#
|
|
|
|
# This option allows configuring the removal of rules_cc in cases where a
|
|
|
|
# project depends on it via an external dependency.
|
|
|
|
#
|
|
|
|
# [1]: https://github.com/bazelbuild/rules_cc
|
|
|
|
, removeRulesCC ? true
|
2019-09-23 13:20:59 +01:00
|
|
|
, removeLocalConfigCc ? true
|
|
|
|
, removeLocal ? true
|
2019-11-24 04:25:02 +00:00
|
|
|
, ...
|
|
|
|
}:
|
2017-11-10 17:06:26 +00:00
|
|
|
|
|
|
|
let
|
2019-11-24 04:25:02 +00:00
|
|
|
fArgs = removeAttrs args [ "buildAttrs" "fetchAttrs" "removeRulesCC" ];
|
2017-11-10 17:06:26 +00:00
|
|
|
fBuildAttrs = fArgs // buildAttrs;
|
|
|
|
fFetchAttrs = fArgs // removeAttrs fetchAttrs [ "sha256" ];
|
|
|
|
|
|
|
|
in stdenv.mkDerivation (fBuildAttrs // {
|
2019-07-18 12:49:00 +01:00
|
|
|
inherit name bazelFlags bazelBuildFlags bazelFetchFlags bazelTarget;
|
2017-11-10 17:06:26 +00:00
|
|
|
|
|
|
|
deps = stdenv.mkDerivation (fFetchAttrs // {
|
|
|
|
name = "${name}-deps";
|
2019-07-18 12:49:00 +01:00
|
|
|
inherit bazelFlags bazelBuildFlags bazelFetchFlags bazelTarget;
|
2017-11-10 17:06:26 +00:00
|
|
|
|
|
|
|
nativeBuildInputs = fFetchAttrs.nativeBuildInputs or [] ++ [ bazel ];
|
|
|
|
|
|
|
|
preHook = fFetchAttrs.preHook or "" + ''
|
2018-07-09 23:38:45 +01:00
|
|
|
export bazelOut="$(echo ''${NIX_BUILD_TOP}/output | sed -e 's,//,/,g')"
|
|
|
|
export bazelUserRoot="$(echo ''${NIX_BUILD_TOP}/tmp | sed -e 's,//,/,g')"
|
2017-11-10 17:06:26 +00:00
|
|
|
export HOME="$NIX_BUILD_TOP"
|
buildBazelPackage: set $USER environment variable (#74538)
tensorflow assumes $USER to be set to something, otherwise it complains
like this:
```
FATAL: $USER is not set, and unable to look up name of current user: (error: 0): Success
Traceback (most recent call last):
File "./configure.py", line 1602, in <module>
main()
File "./configure.py", line 1399, in main
_TF_MAX_BAZEL_VERSION)
File "./configure.py", line 478, in check_bazel_version
['bazel', '--batch', '--bazelrc=/dev/null', 'version'])
File "./configure.py", line 156, in run_shell
output = subprocess.check_output(cmd)
File "/nix/store/drr8qcgiccfc5by09r5zc30flgwh1mbx-python3-3.7.5/lib/python3.7/subprocess.py", line 411, in check_output
**kwargs).stdout
File "/nix/store/drr8qcgiccfc5by09r5zc30flgwh1mbx-python3-3.7.5/lib/python3.7/subprocess.py", line 512, in run
output=stdout, stderr=stderr)
```
Spotted while changing the hash of its fixed-output derivation on
purpose.
We could also set this in the tensorflow-specific part, but very likely,
other programs will fail as well.
2019-11-29 00:14:45 +00:00
|
|
|
export USER="nix"
|
2019-02-01 06:30:36 +00:00
|
|
|
# This is needed for git_repository with https remotes
|
|
|
|
export GIT_SSL_CAINFO="${cacert}/etc/ssl/certs/ca-bundle.crt"
|
2019-11-28 21:39:20 +00:00
|
|
|
# This is needed for Bazel fetchers that are themselves programs (e.g.
|
|
|
|
# rules_go using the go toolchain)
|
|
|
|
export SSL_CERT_FILE="${cacert}/etc/ssl/certs/ca-bundle.crt"
|
2017-11-10 17:06:26 +00:00
|
|
|
'';
|
|
|
|
|
|
|
|
buildPhase = fFetchAttrs.buildPhase or ''
|
|
|
|
runHook preBuild
|
|
|
|
|
2018-09-11 06:30:22 +01:00
|
|
|
# Bazel computes the default value of output_user_root before parsing the
|
|
|
|
# flag. The computation of the default value involves getting the $USER
|
|
|
|
# from the environment. I don't have that variable when building with
|
|
|
|
# sandbox enabled. Code here
|
|
|
|
# https://github.com/bazelbuild/bazel/blob/9323c57607d37f9c949b60e293b573584906da46/src/main/cpp/startup_options.cc#L123-L124
|
|
|
|
#
|
2019-02-19 16:44:05 +00:00
|
|
|
# On macOS Bazel will use the system installed Xcode or CLT toolchain instead of the one in the PATH unless we pass BAZEL_USE_CPP_ONLY_TOOLCHAIN
|
2019-06-20 19:01:22 +01:00
|
|
|
|
|
|
|
# We disable multithreading for the fetching phase since it can lead to timeouts with many dependencies/threads:
|
|
|
|
# https://github.com/bazelbuild/bazel/issues/6502
|
2019-07-13 20:40:22 +01:00
|
|
|
BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 \
|
|
|
|
USER=homeless-shelter \
|
|
|
|
bazel \
|
|
|
|
--output_base="$bazelOut" \
|
|
|
|
--output_user_root="$bazelUserRoot" \
|
|
|
|
fetch \
|
|
|
|
--loading_phase_threads=1 \
|
|
|
|
$bazelFlags \
|
2019-07-18 12:49:00 +01:00
|
|
|
$bazelFetchFlags \
|
2019-07-13 20:40:22 +01:00
|
|
|
$bazelTarget
|
2017-11-10 17:06:26 +00:00
|
|
|
|
|
|
|
runHook postBuild
|
|
|
|
'';
|
|
|
|
|
|
|
|
installPhase = fFetchAttrs.installPhase or ''
|
|
|
|
runHook preInstall
|
|
|
|
|
2018-09-11 06:36:11 +01:00
|
|
|
# Remove all built in external workspaces, Bazel will recreate them when building
|
|
|
|
rm -rf $bazelOut/external/{bazel_tools,\@bazel_tools.marker}
|
2019-11-24 04:25:02 +00:00
|
|
|
${if removeRulesCC then "rm -rf $bazelOut/external/{rules_cc,\\@rules_cc.marker}" else ""}
|
2018-09-11 06:36:11 +01:00
|
|
|
rm -rf $bazelOut/external/{embedded_jdk,\@embedded_jdk.marker}
|
2019-09-23 13:20:59 +01:00
|
|
|
${if removeLocalConfigCc then "rm -rf $bazelOut/external/{local_config_cc,\@local_config_cc.marker}" else ""}
|
|
|
|
${if removeLocal then "rm -rf $bazelOut/external/{local_*,\@local_*.marker}" else ""}
|
2018-09-11 06:36:11 +01:00
|
|
|
|
2019-07-25 12:18:50 +01:00
|
|
|
# Clear markers
|
|
|
|
find $bazelOut/external -name '@*\.marker' -exec sh -c 'echo > {}' \;
|
2018-09-11 06:36:11 +01:00
|
|
|
|
2018-09-11 22:12:31 +01:00
|
|
|
# Remove all vcs files
|
|
|
|
rm -rf $(find $bazelOut/external -type d -name .git)
|
|
|
|
rm -rf $(find $bazelOut/external -type d -name .svn)
|
|
|
|
rm -rf $(find $bazelOut/external -type d -name .hg)
|
2017-11-10 17:06:26 +00:00
|
|
|
|
2019-08-19 09:21:37 +01:00
|
|
|
# Removing top-level symlinks along with their markers.
|
|
|
|
# This is needed because they sometimes point to temporary paths (?).
|
|
|
|
# For example, in Tensorflow-gpu build:
|
|
|
|
# platforms -> NIX_BUILD_TOP/tmp/install/35282f5123611afa742331368e9ae529/_embedded_binaries/platforms
|
|
|
|
find $bazelOut/external -maxdepth 1 -type l | while read symlink; do
|
|
|
|
name="$(basename "$symlink")"
|
|
|
|
rm "$symlink" "$bazelOut/external/@$name.marker"
|
|
|
|
done
|
|
|
|
|
2018-09-11 06:36:11 +01:00
|
|
|
# Patching symlinks to remove build directory reference
|
|
|
|
find $bazelOut/external -type l | while read symlink; do
|
2019-06-20 17:35:16 +01:00
|
|
|
new_target="$(readlink "$symlink" | sed "s,$NIX_BUILD_TOP,NIX_BUILD_TOP,")"
|
|
|
|
rm "$symlink"
|
|
|
|
ln -sf "$new_target" "$symlink"
|
2018-09-11 06:36:11 +01:00
|
|
|
done
|
|
|
|
|
2017-11-10 17:06:26 +00:00
|
|
|
cp -r $bazelOut/external $out
|
|
|
|
|
2020-02-19 13:50:12 +00:00
|
|
|
echo '${bazel.name}' > $out/.nix-bazel-version
|
|
|
|
|
2017-11-10 17:06:26 +00:00
|
|
|
runHook postInstall
|
|
|
|
'';
|
|
|
|
|
|
|
|
dontFixup = true;
|
2019-07-13 20:40:22 +01:00
|
|
|
allowedRequisites = [];
|
|
|
|
|
2017-11-10 17:06:26 +00:00
|
|
|
outputHashMode = "recursive";
|
|
|
|
outputHashAlgo = "sha256";
|
|
|
|
outputHash = fetchAttrs.sha256;
|
|
|
|
});
|
|
|
|
|
2019-07-13 20:40:22 +01:00
|
|
|
nativeBuildInputs = fBuildAttrs.nativeBuildInputs or [] ++ [ (bazel.override { enableNixHacks = true; }) ];
|
2017-11-10 17:06:26 +00:00
|
|
|
|
|
|
|
preHook = fBuildAttrs.preHook or "" + ''
|
|
|
|
export bazelOut="$NIX_BUILD_TOP/output"
|
2018-07-09 23:38:45 +01:00
|
|
|
export bazelUserRoot="$NIX_BUILD_TOP/tmp"
|
2017-11-10 17:06:26 +00:00
|
|
|
export HOME="$NIX_BUILD_TOP"
|
|
|
|
'';
|
|
|
|
|
|
|
|
preConfigure = ''
|
2019-06-26 14:23:50 +01:00
|
|
|
mkdir -p "$bazelOut"
|
2020-02-19 13:50:12 +00:00
|
|
|
|
|
|
|
test "${bazel.name}" = "$(<$deps/.nix-bazel-version)" || {
|
|
|
|
echo "fixed output derivation was built for a different bazel version" >&2
|
|
|
|
echo " got: $(<$deps/.nix-bazel-version)" >&2
|
|
|
|
echo "expected: ${bazel.name}" >&2
|
|
|
|
exit 1
|
|
|
|
}
|
|
|
|
|
2019-06-26 14:23:50 +01:00
|
|
|
cp -r $deps $bazelOut/external
|
2017-11-10 17:06:26 +00:00
|
|
|
chmod -R +w $bazelOut
|
|
|
|
find $bazelOut -type l | while read symlink; do
|
|
|
|
ln -sf $(readlink "$symlink" | sed "s,NIX_BUILD_TOP,$NIX_BUILD_TOP,") "$symlink"
|
|
|
|
done
|
|
|
|
'' + fBuildAttrs.preConfigure or "";
|
|
|
|
|
|
|
|
buildPhase = fBuildAttrs.buildPhase or ''
|
|
|
|
runHook preBuild
|
|
|
|
|
2019-07-25 10:43:45 +01:00
|
|
|
# Bazel sandboxes the execution of the tools it invokes, so even though we are
|
|
|
|
# calling the correct nix wrappers, the values of the environment variables
|
|
|
|
# the wrappers are expecting will not be set. So instead of relying on the
|
|
|
|
# wrappers picking them up, pass them in explicitly via `--copt`, `--linkopt`
|
|
|
|
# and related flags.
|
|
|
|
#
|
|
|
|
copts=()
|
|
|
|
host_copts=()
|
|
|
|
for flag in $NIX_CFLAGS_COMPILE; do
|
|
|
|
copts+=( "--copt=$flag" )
|
|
|
|
host_copts+=( "--host_copt=$flag" )
|
|
|
|
done
|
|
|
|
for flag in $NIX_CXXSTDLIB_COMPILE; do
|
|
|
|
copts+=( "--copt=$flag" )
|
|
|
|
host_copts+=( "--host_copt=$flag" )
|
|
|
|
done
|
|
|
|
linkopts=()
|
|
|
|
host_linkopts=()
|
|
|
|
for flag in $NIX_LD_FLAGS; do
|
|
|
|
linkopts+=( "--linkopt=$flag" )
|
|
|
|
host_linkopts+=( "--host_linkopt=$flag" )
|
|
|
|
done
|
|
|
|
|
2019-02-19 16:44:05 +00:00
|
|
|
BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 \
|
|
|
|
USER=homeless-shelter \
|
|
|
|
bazel \
|
|
|
|
--output_base="$bazelOut" \
|
|
|
|
--output_user_root="$bazelUserRoot" \
|
|
|
|
build \
|
|
|
|
-j $NIX_BUILD_CORES \
|
2019-07-25 10:43:45 +01:00
|
|
|
"''${copts[@]}" \
|
|
|
|
"''${host_copts[@]}" \
|
|
|
|
"''${linkopts[@]}" \
|
|
|
|
"''${host_linkopts[@]}" \
|
2019-02-19 16:44:05 +00:00
|
|
|
$bazelFlags \
|
2019-07-18 12:49:00 +01:00
|
|
|
$bazelBuildFlags \
|
2019-02-19 16:44:05 +00:00
|
|
|
$bazelTarget
|
2017-11-10 17:06:26 +00:00
|
|
|
|
|
|
|
runHook postBuild
|
|
|
|
'';
|
|
|
|
})
|