3
0
Fork 0
forked from mirrors/nixpkgs
nixpkgs/pkgs/stdenv/generic/make-derivation.nix
Graham Christensen 817c933878
check-env: don't execute check-meta.nix 15,000 times
Generated from https://github.com/NixOS/nix/pull/2761:

```
                                                                                          ns     calls ns/call
- /home/grahamc/projects/github.com/NixOS/nixpkgs/pkgs/stdenv/generic/check-meta.nix:22:5 591200 15026 39.3451
+ /home/grahamc/projects/github.com/NixOS/nixpkgs/pkgs/stdenv/generic/check-meta.nix:22:5 8744   308   28.3896
```

more, generated by:

```
$ NIX_SHOW_STATS=1 NIX_COUNT_CALLS=1 nix-instantiate ./pkgs/top-level/release.nix -A unstable > before 2>&1
$ jq -r '.functions | map((.name + ":" + .file + ":" + (.line|tostring) + ":" + (.column|tostring) + " " + (.count|tostring))) | .[]' before | sort  > before.list
```

applying this patch, then:

```
$ NIX_SHOW_STATS=1 NIX_COUNT_CALLS=1 nix-instantiate ./pkgs/top-level/release.nix -A unstable > after 2>&1
$ jq -r '.functions | map((.name + ":" + .file + ":" + (.line|tostring) + ":" + (.column|tostring) + " " + (.count|tostring))) | .[]' after | sort  > after.list
```

and then diffing before.list and after.list to get:

```
                                                                                                        calls
- :/home/grahamc/projects/github.com/NixOS/nixpkgs/pkgs/stdenv/generic/check-meta.nix:4:1               7513
+ :/home/grahamc/projects/github.com/NixOS/nixpkgs/pkgs/stdenv/generic/check-meta.nix:4:1               154

- mutuallyExclusive:/home/grahamc/projects/github.com/NixOS/nixpkgs/lib/lists.nix:658:23                7513
+ mutuallyExclusive:/home/grahamc/projects/github.com/NixOS/nixpkgs/lib/lists.nix:658:23                154

- mutuallyExclusive:/home/grahamc/projects/github.com/NixOS/nixpkgs/lib/lists.nix:658:26                7513
+ mutuallyExclusive:/home/grahamc/projects/github.com/NixOS/nixpkgs/lib/lists.nix:658:26                154

- onlyLicenses:/home/grahamc/projects/github.com/NixOS/nixpkgs/pkgs/stdenv/generic/check-meta.nix:21:18 15026
+ onlyLicenses:/home/grahamc/projects/github.com/NixOS/nixpkgs/pkgs/stdenv/generic/check-meta.nix:21:18 308
```

The following information is from `NIX_SHOW_STATS=1 GC_INITIAL_HEAP_SIZE=4g nix-env -f ./outpaths.nix -qaP --no-name --out-path --arg checkMeta true`:

| stat                       | before         | after          | Δ               | Δ%      |
|:---------------------------|---------------:|---------------:|:----------------|--------:|
| **cpuTime**                |        179.915 |        145.543 | 🡖 34.372        | -19.10% |
| **envs-bytes**             |  3,900,878,824 |  3,599,483,208 | 🡖 301,395,616   |  -7.73% |
| **envs-elements**          |    214,426,071 |    185,881,709 | 🡖 28,544,362    | -13.31% |
| **envs-number**            |    136,591,891 |    132,026,846 | 🡖 4,565,045     |  -3.34% |
| **gc-heapSize**            | 11,400,048,640 | 12,314,890,240 | 🡕 914,841,600   |   8.02% |
| **gc-totalBytes**          | 25,976,902,560 | 24,510,740,176 | 🡖 1,466,162,384 |  -5.64% |
| **list-bytes**             |  1,665,290,080 |  1,665,290,080 | 0               |         |
| **list-concats**           |      7,264,417 |      7,264,417 | 0               |         |
| **list-elements**          |    208,161,260 |    208,161,260 | 0               |         |
| **nrAvoided**              |    191,359,386 |    179,693,661 | 🡖 11,665,725    |  -6.10% |
| **nrFunctionCalls**        |    119,665,062 |    116,348,547 | 🡖 3,316,515     |  -2.77% |
| **nrLookups**              |     80,996,257 |     76,069,825 | 🡖 4,926,432     |  -6.08% |
| **nrOpUpdateValuesCopied** |    213,930,649 |    213,930,649 | 0               |         |
| **nrOpUpdates**            |     12,025,937 |     12,025,937 | 0               |         |
| **nrPrimOpCalls**          |     88,105,604 |     86,451,598 | 🡖 1,654,006     |  -1.88% |
| **nrThunks**               |    196,842,044 |    175,126,701 | 🡖 21,715,343    | -11.03% |
| **sets-bytes**             |  7,678,425,776 |  7,285,767,928 | 🡖 392,657,848   |  -5.11% |
| **sets-elements**          |    310,241,340 |    294,373,227 | 🡖 15,868,113    |  -5.11% |
| **sets-number**            |     29,079,202 |     27,601,310 | 🡖 1,477,892     |  -5.08% |
| **sizes-Attr**             |             24 |             24 | 0               |         |
| **sizes-Bindings**         |              8 |              8 | 0               |         |
| **sizes-Env**              |             16 |             16 | 0               |         |
| **sizes-Value**            |             24 |             24 | 0               |         |
| **symbols-bytes**          |     16,474,666 |     16,474,676 | 🡕 10            |   0.00% |
| **symbols-number**         |        376,426 |        376,427 | 🡕 1             |   0.00% |
| **values-bytes**           |  6,856,506,288 |  6,316,585,560 | 🡖 539,920,728   |  -7.87% |
| **values-number**          |    285,687,762 |    263,191,065 | 🡖 22,496,697    |  -7.87% |

The following information is from `NIX_SHOW_STATS=1 GC_INITIAL_HEAP_SIZE=4g nix-instantiate ./nixos/release-combined.nix -A tested`:

| stat                       | before         | after          | Δ               | Δ%     |
|:---------------------------|---------------:|---------------:|:----------------|-------:|
| **cpuTime**                |        256.071 |        237.531 | 🡖 18.54         | -7.24% |
| **envs-bytes**             |  7,111,004,192 |  7,041,478,520 | 🡖 69,525,672    | -0.98% |
| **envs-elements**          |    346,236,940 |    339,588,487 | 🡖 6,648,453     | -1.92% |
| **envs-number**            |    271,319,292 |    270,298,164 | 🡖 1,021,128     | -0.38% |
| **gc-heapSize**            |  8,995,291,136 | 10,110,009,344 | 🡕 1,114,718,208 | 12.39% |
| **gc-totalBytes**          | 37,172,737,408 | 36,878,391,888 | 🡖 294,345,520   | -0.79% |
| **list-bytes**             |  1,886,162,656 |  1,886,163,472 | 🡕 816           |  0.00% |
| **list-concats**           |      6,898,114 |      6,898,114 | 0               |        |
| **list-elements**          |    235,770,332 |    235,770,434 | 🡕 102           |  0.00% |
| **nrAvoided**              |    328,829,821 |    326,618,157 | 🡖 2,211,664     | -0.67% |
| **nrFunctionCalls**        |    240,850,845 |    239,998,495 | 🡖 852,350       | -0.35% |
| **nrLookups**              |    144,849,632 |    142,126,339 | 🡖 2,723,293     | -1.88% |
| **nrOpUpdateValuesCopied** |    251,032,504 |    251,032,504 | 0               |        |
| **nrOpUpdates**            |     17,903,110 |     17,903,110 | 0               |        |
| **nrPrimOpCalls**          |    140,674,913 |    139,485,975 | 🡖 1,188,938     | -0.85% |
| **nrThunks**               |    294,643,131 |    288,678,022 | 🡖 5,965,109     | -2.02% |
| **sets-bytes**             |  9,464,322,192 |  9,456,172,048 | 🡖 8,150,144     | -0.09% |
| **sets-elements**          |    377,474,889 |    377,134,877 | 🡖 340,012       | -0.09% |
| **sets-number**            |     50,615,607 |     50,616,875 | 🡕 1,268         |  0.00% |
| **sizes-Attr**             |             24 |             24 | 0               |        |
| **sizes-Bindings**         |              8 |              8 | 0               |        |
| **sizes-Env**              |             16 |             16 | 0               |        |
| **sizes-Value**            |             24 |             24 | 0               |        |
| **symbols-bytes**          |      3,147,102 |      3,147,064 | 🡖 38            | -0.00% |
| **symbols-number**         |         82,819 |         82,819 | 0               |        |
| **values-bytes**           | 11,147,448,768 | 10,996,111,512 | 🡖 151,337,256   | -1.36% |
| **values-number**          |    464,477,032 |    458,171,313 | 🡖 6,305,719     | -1.36% |
2019-04-11 19:21:38 -04:00

323 lines
15 KiB
Nix

{ lib, config, stdenv }:
let
checkMeta = import ./check-meta.nix {
inherit lib config;
# Nix itself uses the `system` field of a derivation to decide where
# to build it. This is a bit confusing for cross compilation.
inherit (stdenv) hostPlatform;
};
in rec {
# `mkDerivation` wraps the builtin `derivation` function to
# produce derivations that use this stdenv and its shell.
#
# See also:
#
# * https://nixos.org/nixpkgs/manual/#sec-using-stdenv
# Details on how to use this mkDerivation function
#
# * https://nixos.org/nix/manual/#ssec-derivation
# Explanation about derivations in general
mkDerivation =
{
# These types of dependencies are all exhaustively documented in
# the "Specifying Dependencies" section of the "Standard
# Environment" chapter of the Nixpkgs manual.
# TODO(@Ericson2314): Stop using legacy dep attribute names
# host offset -> target offset
depsBuildBuild ? [] # -1 -> -1
, depsBuildBuildPropagated ? [] # -1 -> -1
, nativeBuildInputs ? [] # -1 -> 0 N.B. Legacy name
, propagatedNativeBuildInputs ? [] # -1 -> 0 N.B. Legacy name
, depsBuildTarget ? [] # -1 -> 1
, depsBuildTargetPropagated ? [] # -1 -> 1
, depsHostHost ? [] # 0 -> 0
, depsHostHostPropagated ? [] # 0 -> 0
, buildInputs ? [] # 0 -> 1 N.B. Legacy name
, propagatedBuildInputs ? [] # 0 -> 1 N.B. Legacy name
, depsTargetTarget ? [] # 1 -> 1
, depsTargetTargetPropagated ? [] # 1 -> 1
, checkInputs ? []
, installCheckInputs ? []
# Configure Phase
, configureFlags ? []
, cmakeFlags ? []
, # Target is not included by default because most programs don't care.
# Including it then would cause needless mass rebuilds.
#
# TODO(@Ericson2314): Make [ "build" "host" ] always the default.
configurePlatforms ? lib.optionals
(stdenv.hostPlatform != stdenv.buildPlatform)
[ "build" "host" ]
# TODO(@Ericson2314): Make unconditional / resolve #33599
# Check phase
, doCheck ? config.doCheckByDefault or false
# TODO(@Ericson2314): Make unconditional / resolve #33599
# InstallCheck phase
, doInstallCheck ? config.doCheckByDefault or false
, # TODO(@Ericson2314): Make always true and remove
strictDeps ? stdenv.hostPlatform != stdenv.buildPlatform
, meta ? {}
, passthru ? {}
, pos ? # position used in error messages and for meta.position
(if attrs.meta.description or null != null
then builtins.unsafeGetAttrPos "description" attrs.meta
else if attrs.version or null != null
then builtins.unsafeGetAttrPos "version" attrs
else builtins.unsafeGetAttrPos "name" attrs)
, separateDebugInfo ? false
, outputs ? [ "out" ]
, __darwinAllowLocalNetworking ? false
, __impureHostDeps ? []
, __propagatedImpureHostDeps ? []
, sandboxProfile ? ""
, propagatedSandboxProfile ? ""
, hardeningEnable ? []
, hardeningDisable ? []
, ... } @ attrs:
let
# TODO(@oxij, @Ericson2314): This is here to keep the old semantics, remove when
# no package has `doCheck = true`.
doCheck' = doCheck && stdenv.hostPlatform == stdenv.buildPlatform;
doInstallCheck' = doInstallCheck && stdenv.hostPlatform == stdenv.buildPlatform;
separateDebugInfo' = separateDebugInfo && stdenv.hostPlatform.isLinux && !(stdenv.hostPlatform.useLLVM or false);
outputs' = outputs ++ lib.optional separateDebugInfo' "debug";
fixedOutputDrv = attrs ? outputHash;
noNonNativeDeps = builtins.length (depsBuildTarget ++ depsBuildTargetPropagated
++ depsHostHost ++ depsHostHostPropagated
++ buildInputs ++ propagatedBuildInputs
++ depsTargetTarget ++ depsTargetTargetPropagated) == 0;
dontAddHostSuffix = attrs ? outputHash && !noNonNativeDeps || (stdenv.noCC or false);
supportedHardeningFlags = [ "fortify" "stackprotector" "pie" "pic" "strictoverflow" "format" "relro" "bindnow" ];
defaultHardeningFlags = if stdenv.hostPlatform.isMusl
then supportedHardeningFlags
else lib.remove "pie" supportedHardeningFlags;
enabledHardeningOptions =
if builtins.elem "all" hardeningDisable
then []
else lib.subtractLists hardeningDisable (defaultHardeningFlags ++ hardeningEnable);
# hardeningDisable additionally supports "all".
erroneousHardeningFlags = lib.subtractLists supportedHardeningFlags (hardeningEnable ++ lib.remove "all" hardeningDisable);
in if builtins.length erroneousHardeningFlags != 0
then abort ("mkDerivation was called with unsupported hardening flags: " + lib.generators.toPretty {} {
inherit erroneousHardeningFlags hardeningDisable hardeningEnable supportedHardeningFlags;
})
else let
doCheck = doCheck';
doInstallCheck = doInstallCheck';
outputs = outputs';
references = nativeBuildInputs ++ buildInputs
++ propagatedNativeBuildInputs ++ propagatedBuildInputs;
dependencies = map (map lib.chooseDevOutputs) [
[
(map (drv: drv.__spliced.buildBuild or drv) depsBuildBuild)
(map (drv: drv.nativeDrv or drv) nativeBuildInputs
++ lib.optional separateDebugInfo' ../../build-support/setup-hooks/separate-debug-info.sh
++ lib.optional stdenv.hostPlatform.isWindows ../../build-support/setup-hooks/win-dll-link.sh
++ lib.optionals doCheck checkInputs
++ lib.optionals doInstallCheck' installCheckInputs)
(map (drv: drv.__spliced.buildTarget or drv) depsBuildTarget)
]
[
(map (drv: drv.__spliced.hostHost or drv) depsHostHost)
(map (drv: drv.crossDrv or drv) buildInputs)
]
[
(map (drv: drv.__spliced.targetTarget or drv) depsTargetTarget)
]
];
propagatedDependencies = map (map lib.chooseDevOutputs) [
[
(map (drv: drv.__spliced.buildBuild or drv) depsBuildBuildPropagated)
(map (drv: drv.nativeDrv or drv) propagatedNativeBuildInputs)
(map (drv: drv.__spliced.buildTarget or drv) depsBuildTargetPropagated)
]
[
(map (drv: drv.__spliced.hostHost or drv) depsHostHostPropagated)
(map (drv: drv.crossDrv or drv) propagatedBuildInputs)
]
[
(map (drv: drv.__spliced.targetTarget or drv) depsTargetTargetPropagated)
]
];
computedSandboxProfile =
lib.concatMap (input: input.__propagatedSandboxProfile or [])
(stdenv.extraNativeBuildInputs
++ stdenv.extraBuildInputs
++ lib.concatLists dependencies);
computedPropagatedSandboxProfile =
lib.concatMap (input: input.__propagatedSandboxProfile or [])
(lib.concatLists propagatedDependencies);
computedImpureHostDeps =
lib.unique (lib.concatMap (input: input.__propagatedImpureHostDeps or [])
(stdenv.extraNativeBuildInputs
++ stdenv.extraBuildInputs
++ lib.concatLists dependencies));
computedPropagatedImpureHostDeps =
lib.unique (lib.concatMap (input: input.__propagatedImpureHostDeps or [])
(lib.concatLists propagatedDependencies));
derivationArg =
(removeAttrs attrs
["meta" "passthru" "pos"
"checkInputs" "installCheckInputs"
"__darwinAllowLocalNetworking"
"__impureHostDeps" "__propagatedImpureHostDeps"
"sandboxProfile" "propagatedSandboxProfile"])
// (lib.optionalAttrs (!(attrs ? name) && attrs ? pname && attrs ? version)) {
name = "${attrs.pname}-${attrs.version}";
} // (lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform && !dontAddHostSuffix && (attrs ? name || (attrs ? pname && attrs ? version)))) {
# Fixed-output derivations like source tarballs shouldn't get a host
# suffix. But we have some weird ones with run-time deps that are
# just used for their side-affects. Those might as well since the
# hash can't be the same. See #32986.
name = "${attrs.name or "${attrs.pname}-${attrs.version}"}-${stdenv.hostPlatform.config}";
} // {
builder = attrs.realBuilder or stdenv.shell;
args = attrs.args or ["-e" (attrs.builder or ./default-builder.sh)];
inherit stdenv;
# The `system` attribute of a derivation has special meaning to Nix.
# Derivations set it to choose what sort of machine could be used to
# execute the build, The build platform entirely determines this,
# indeed more finely than Nix knows or cares about. The `system`
# attribute of `buildPlatfom` matches Nix's degree of specificity.
# exactly.
inherit (stdenv.buildPlatform) system;
userHook = config.stdenv.userHook or null;
__ignoreNulls = true;
inherit strictDeps;
depsBuildBuild = lib.elemAt (lib.elemAt dependencies 0) 0;
nativeBuildInputs = lib.elemAt (lib.elemAt dependencies 0) 1;
depsBuildTarget = lib.elemAt (lib.elemAt dependencies 0) 2;
depsHostHost = lib.elemAt (lib.elemAt dependencies 1) 0;
buildInputs = lib.elemAt (lib.elemAt dependencies 1) 1;
depsTargetTarget = lib.elemAt (lib.elemAt dependencies 2) 0;
depsBuildBuildPropagated = lib.elemAt (lib.elemAt propagatedDependencies 0) 0;
propagatedNativeBuildInputs = lib.elemAt (lib.elemAt propagatedDependencies 0) 1;
depsBuildTargetPropagated = lib.elemAt (lib.elemAt propagatedDependencies 0) 2;
depsHostHostPropagated = lib.elemAt (lib.elemAt propagatedDependencies 1) 0;
propagatedBuildInputs = lib.elemAt (lib.elemAt propagatedDependencies 1) 1;
depsTargetTargetPropagated = lib.elemAt (lib.elemAt propagatedDependencies 2) 0;
# This parameter is sometimes a string, sometimes null, and sometimes a list, yuck
configureFlags = let inherit (lib) optional elem; in
(/**/ if lib.isString configureFlags then [configureFlags]
else if configureFlags == null then []
else configureFlags)
++ optional (elem "build" configurePlatforms) "--build=${stdenv.buildPlatform.config}"
++ optional (elem "host" configurePlatforms) "--host=${stdenv.hostPlatform.config}"
++ optional (elem "target" configurePlatforms) "--target=${stdenv.targetPlatform.config}";
inherit doCheck doInstallCheck;
inherit outputs;
} // lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform) {
cmakeFlags =
(/**/ if lib.isString cmakeFlags then [cmakeFlags]
else if cmakeFlags == null then []
else cmakeFlags)
++ lib.optional (stdenv.hostPlatform.uname.system != null) "-DCMAKE_SYSTEM_NAME=${stdenv.hostPlatform.uname.system}"
++ lib.optional (stdenv.hostPlatform.uname.processor != null) "-DCMAKE_SYSTEM_PROCESSOR=${stdenv.hostPlatform.uname.processor}"
++ lib.optional (stdenv.hostPlatform.uname.release != null) "-DCMAKE_SYSTEM_VERSION=${stdenv.hostPlatform.release}"
++ lib.optional (stdenv.buildPlatform.uname.system != null) "-DCMAKE_HOST_SYSTEM_NAME=${stdenv.buildPlatform.uname.system}"
++ lib.optional (stdenv.buildPlatform.uname.processor != null) "-DCMAKE_HOST_SYSTEM_PROCESSOR=${stdenv.buildPlatform.uname.processor}"
++ lib.optional (stdenv.buildPlatform.uname.release != null) "-DCMAKE_HOST_SYSTEM_VERSION=${stdenv.buildPlatform.uname.release}";
} // lib.optionalAttrs (attrs.enableParallelBuilding or false) {
enableParallelChecking = attrs.enableParallelChecking or true;
} // lib.optionalAttrs (hardeningDisable != [] || hardeningEnable != []) {
NIX_HARDENING_ENABLE = enabledHardeningOptions;
} // lib.optionalAttrs (stdenv.buildPlatform.isDarwin) {
inherit __darwinAllowLocalNetworking;
# TODO: remove lib.unique once nix has a list canonicalization primitive
__sandboxProfile =
let profiles = [ stdenv.extraSandboxProfile ] ++ computedSandboxProfile ++ computedPropagatedSandboxProfile ++ [ propagatedSandboxProfile sandboxProfile ];
final = lib.concatStringsSep "\n" (lib.filter (x: x != "") (lib.unique profiles));
in final;
__propagatedSandboxProfile = lib.unique (computedPropagatedSandboxProfile ++ [ propagatedSandboxProfile ]);
__impureHostDeps = computedImpureHostDeps ++ computedPropagatedImpureHostDeps ++ __propagatedImpureHostDeps ++ __impureHostDeps ++ stdenv.__extraImpureHostDeps ++ [
"/dev/zero"
"/dev/random"
"/dev/urandom"
"/bin/sh"
];
__propagatedImpureHostDeps = computedPropagatedImpureHostDeps ++ __propagatedImpureHostDeps;
};
validity = checkMeta { inherit meta attrs; };
# The meta attribute is passed in the resulting attribute set,
# but it's not part of the actual derivation, i.e., it's not
# passed to the builder and is not a dependency. But since we
# include it in the result, it *is* available to nix-env for queries.
meta = {
# `name` above includes cross-compilation cruft (and is under assert),
# lets have a clean always accessible version here.
name = attrs.name or "${attrs.pname}-${attrs.version}";
# If the packager hasn't specified `outputsToInstall`, choose a default,
# which is the name of `p.bin or p.out or p`;
# if he has specified it, it will be overridden below in `// meta`.
# Note: This default probably shouldn't be globally configurable.
# Services and users should specify outputs explicitly,
# unless they are comfortable with this default.
outputsToInstall =
let
hasOutput = out: builtins.elem out outputs;
in [( lib.findFirst hasOutput null (["bin" "out"] ++ outputs) )]
++ lib.optional (hasOutput "man") "man";
}
// attrs.meta or {}
# Fill `meta.position` to identify the source location of the package.
// lib.optionalAttrs (pos != null) {
position = pos.file + ":" + toString pos.line;
# Expose the result of the checks for everyone to see.
} // {
available = validity.valid
&& (if config.checkMetaRecursively or false
then lib.all (d: d.meta.available or true) references
else true);
};
in
lib.extendDerivation
validity.handled
({
overrideAttrs = f: mkDerivation (attrs // (f attrs));
inherit meta passthru;
} //
# Pass through extra attributes that are not inputs, but
# should be made available to Nix expressions using the
# derivation (e.g., in assertions).
passthru)
(derivation derivationArg);
}