alioth/nixpkgs
3
0
Fork 0
forked from mirrors/nixpkgs
Code Releases Activity

pythonPackages: ensure all derivations provide python modules

Browse source 
This adds a test to ensure no new uses of `buildPythonApplication` can
be added to `python-packages.nix`.

Python packages can be grouped into two groups: 1) applications and 2)
packages providing importable modules. In `python-packages.nix` we only
want to have 2). 1) should be in the top-level package set.

To achieve this, all setup hooks need to be marked as being a setup hook.
For the setup hooks in the Python packages set this is done by creating
a new builder, `makePythonHook`.

Because there were issues with splicing, the file importing all the hooks
is converted to an extension. All non-packages were moved out of `python-packages.nix`
into `python-packages-base.nix`. The `keep` argument to `makeScopeWithSplicing
was cleaned up as well; there is no need to keep this one manually in sync
reducing the risk of breaking cross-compilation.
This commit is contained in:
Frederik Rietdijk 2022-10-03 12:25:02 +02:00 committed by Frederik Rietdijk
parent 99bcead8dd
commit 33d12e5f0b
6 changed files with 185 additions and 242 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

62
pkgs/development/interpreters/python/default.nix
View file

@ -31,13 +31,19 @@
, pythonAttr ? null
, self # is pythonOnHostForTarget
}: let
pythonPackages = callPackage
pythonPackages = let
ensurePythonModules = items: let
providesSetupHook = lib.attrByPath [ "provides" "setupHook"] false;
notValid = value: (lib.isDerivation value) && !((pythonPackages.hasPythonModule value) || (providesSetupHook value));
func = name: value: if !(notValid value) then value else throw "${name} should use `buildPythonPackage` or `toPythonModule` if it is to be part of the Python packages set.";
in lib.mapAttrs func items;
in ensurePythonModules (callPackage
# Function that when called
# - imports python-packages.nix
# - adds spliced package sets to the package set
# - applies overrides from `packageOverrides` and `pythonPackagesOverlays`.
({ pkgs, stdenv, python, overrides }: let
pythonPackagesFun = import ../../../top-level/python-packages.nix {
pythonPackagesFun = import ./python-packages-base.nix {
inherit stdenv pkgs lib;
python = self;
};
@ -48,47 +54,19 @@
selfHostHost = pythonOnHostForHost.pkgs;
selfTargetTarget = pythonOnTargetForTarget.pkgs or {}; # There is no Python TargetTarget.
};
keep = self: {
# TODO maybe only define these here so nothing is needed to be kept in sync.
inherit (self)
isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy310 isPy3k isPyPy pythonAtLeast pythonOlder
python bootstrapped-pip buildPythonPackage buildPythonApplication
fetchPypi
hasPythonModule requiredPythonModules makePythonPath disabledIf
toPythonModule toPythonApplication
buildSetupcfg
condaInstallHook
condaUnpackHook
eggUnpackHook
eggBuildHook
eggInstallHook
flitBuildHook
pipBuildHook
pipInstallHook
pytestCheckHook
pythonCatchConflictsHook
pythonImportsCheckHook
pythonNamespacesHook
pythonRecompileBytecodeHook
pythonRemoveBinBytecodeHook
pythonRemoveTestsDirHook
setuptoolsBuildHook
setuptoolsCheckHook
venvShellHook
wheelUnpackHook
wrapPython
pythonPackages
recursivePthLoader
;
};
hooks = import ./hooks/default.nix;
keep = lib.extends hooks pythonPackagesFun;
extra = _: {};
optionalExtensions = cond: as: if cond then as else [];
pythonExtension = import ../../../top-level/python-packages.nix;
python2Extension = import ../../../top-level/python2-packages.nix;
extensions = lib.composeManyExtensions ((optionalExtensions (!self.isPy3k) [python2Extension]) ++ pythonPackagesExtensions ++ [ overrides ]);
extensions = lib.composeManyExtensions ([
pythonExtension
] ++ (optionalExtensions (!self.isPy3k) [
python2Extension
]) ++ pythonPackagesExtensions ++ [
overrides
]);
aliases = self: super: lib.optionalAttrs config.allowAliases (import ../../../top-level/python-aliases.nix lib self super);
in lib.makeScopeWithSplicing
splicePackages
@ -96,11 +74,11 @@
otherSplices
keep
extra
(lib.extends (lib.composeExtensions aliases extensions) pythonPackagesFun))
(lib.extends (lib.composeExtensions aliases extensions) keep))
{
overrides = packageOverrides;
python = self;
};
});
in rec {
isPy27 = pythonVersion == "2.7";
isPy35 = pythonVersion == "3.5";

114
pkgs/development/interpreters/python/hooks/default.nix
View file

@ -1,21 +1,15 @@
# Hooks for building Python packages.
{ python
, lib
, makeSetupHook
, disabledIf
, isPy3k
}:
self: super: with self;
let
callPackage = python.pythonForBuild.pkgs.callPackage;
pythonInterpreter = python.pythonForBuild.interpreter;
pythonSitePackages = python.sitePackages;
pythonCheckInterpreter = python.interpreter;
pythonInterpreter = super.python.pythonForBuild.interpreter;
pythonSitePackages = super.python.sitePackages;
pythonCheckInterpreter = super.python.interpreter;
setuppy = ../run_setup.py;
in rec {
in {
makePythonHook = args: pkgs.makeSetupHook ({passthru.provides.setupHook = true; } // args);
condaInstallHook = callPackage ({ gnutar, lbzip2 }:
makeSetupHook {
condaInstallHook = callPackage ({ makePythonHook, gnutar, lbzip2 }:
makePythonHook {
name = "conda-install-hook";
deps = [ gnutar lbzip2 ];
substitutions = {
@ -23,20 +17,20 @@ in rec {
};
} ./conda-install-hook.sh) {};
condaUnpackHook = callPackage ({}:
makeSetupHook {
condaUnpackHook = callPackage ({ makePythonHook }:
makePythonHook {
name = "conda-unpack-hook";
deps = [];
} ./conda-unpack-hook.sh) {};
eggBuildHook = callPackage ({ }:
makeSetupHook {
eggBuildHook = callPackage ({ makePythonHook }:
makePythonHook {
name = "egg-build-hook.sh";
deps = [ ];
} ./egg-build-hook.sh) {};
eggInstallHook = callPackage ({ setuptools }:
makeSetupHook {
eggInstallHook = callPackage ({ makePythonHook, setuptools }:
makePythonHook {
name = "egg-install-hook.sh";
deps = [ setuptools ];
substitutions = {
@ -44,14 +38,14 @@ in rec {
};
} ./egg-install-hook.sh) {};
eggUnpackHook = callPackage ({ }:
makeSetupHook {
eggUnpackHook = callPackage ({ makePythonHook, }:
makePythonHook {
name = "egg-unpack-hook.sh";
deps = [ ];
} ./egg-unpack-hook.sh) {};
flitBuildHook = callPackage ({ flit }:
makeSetupHook {
flitBuildHook = callPackage ({ makePythonHook, flit }:
makePythonHook {
name = "flit-build-hook";
deps = [ flit ];
substitutions = {
@ -59,8 +53,8 @@ in rec {
};
} ./flit-build-hook.sh) {};
pipBuildHook = callPackage ({ pip, wheel }:
makeSetupHook {
pipBuildHook = callPackage ({ makePythonHook, pip, wheel }:
makePythonHook {
name = "pip-build-hook.sh";
deps = [ pip wheel ];
substitutions = {
@ -68,8 +62,8 @@ in rec {
};
} ./pip-build-hook.sh) {};
pipInstallHook = callPackage ({ pip }:
makeSetupHook {
pipInstallHook = callPackage ({ makePythonHook, pip }:
makePythonHook {
name = "pip-install-hook";
deps = [ pip ];
substitutions = {
@ -77,8 +71,8 @@ in rec {
};
} ./pip-install-hook.sh) {};
pytestCheckHook = callPackage ({ pytest }:
makeSetupHook {
pytestCheckHook = callPackage ({ makePythonHook, pytest }:
makePythonHook {
name = "pytest-check-hook";
deps = [ pytest ];
substitutions = {
@ -86,8 +80,8 @@ in rec {
};
} ./pytest-check-hook.sh) {};
pythonCatchConflictsHook = callPackage ({ setuptools }:
makeSetupHook {
pythonCatchConflictsHook = callPackage ({ makePythonHook, setuptools }:
makePythonHook {
name = "python-catch-conflicts-hook";
substitutions = {
inherit pythonInterpreter pythonSitePackages setuptools;
@ -95,29 +89,29 @@ in rec {
};
} ./python-catch-conflicts-hook.sh) {};
pythonImportsCheckHook = callPackage ({}:
makeSetupHook {
pythonImportsCheckHook = callPackage ({ makePythonHook }:
makePythonHook {
name = "python-imports-check-hook.sh";
substitutions = {
inherit pythonCheckInterpreter;
};
} ./python-imports-check-hook.sh) {};
pythonNamespacesHook = callPackage ({ findutils }:
makeSetupHook {
pythonNamespacesHook = callPackage ({ makePythonHook, findutils }:
makePythonHook {
name = "python-namespaces-hook.sh";
substitutions = {
inherit pythonSitePackages findutils;
};
} ./python-namespaces-hook.sh) {};
pythonOutputDistHook = callPackage ({ }:
makeSetupHook {
pythonOutputDistHook = callPackage ({ makePythonHook }:
makePythonHook {
name = "python-output-dist-hook";
} ./python-output-dist-hook.sh ) {};
pythonRecompileBytecodeHook = callPackage ({ }:
makeSetupHook {
pythonRecompileBytecodeHook = callPackage ({ makePythonHook }:
makePythonHook {
name = "python-recompile-bytecode-hook";
substitutions = {
inherit pythonInterpreter pythonSitePackages;
@ -126,8 +120,8 @@ in rec {
};
} ./python-recompile-bytecode-hook.sh ) {};
pythonRelaxDepsHook = callPackage ({ wheel }:
makeSetupHook {
pythonRelaxDepsHook = callPackage ({ makePythonHook, wheel }:
makePythonHook {
name = "python-relax-deps-hook";
deps = [ wheel ];
substitutions = {
@ -135,21 +129,21 @@ in rec {
};
} ./python-relax-deps-hook.sh) {};
pythonRemoveBinBytecodeHook = callPackage ({ }:
makeSetupHook {
pythonRemoveBinBytecodeHook = callPackage ({ makePythonHook }:
makePythonHook {
name = "python-remove-bin-bytecode-hook";
} ./python-remove-bin-bytecode-hook.sh) {};
pythonRemoveTestsDirHook = callPackage ({ }:
makeSetupHook {
pythonRemoveTestsDirHook = callPackage ({ makePythonHook }:
makePythonHook {
name = "python-remove-tests-dir-hook";
substitutions = {
inherit pythonSitePackages;
};
} ./python-remove-tests-dir-hook.sh) {};
setuptoolsBuildHook = callPackage ({ setuptools, wheel }:
makeSetupHook {
setuptoolsBuildHook = callPackage ({ makePythonHook, setuptools, wheel }:
makePythonHook {
name = "setuptools-setup-hook";
deps = [ setuptools wheel ];
substitutions = {
@ -157,8 +151,8 @@ in rec {
};
} ./setuptools-build-hook.sh) {};
setuptoolsCheckHook = callPackage ({ setuptools }:
makeSetupHook {
setuptoolsCheckHook = callPackage ({ makePythonHook, setuptools }:
makePythonHook {
name = "setuptools-check-hook";
deps = [ setuptools ];
substitutions = {
@ -166,16 +160,16 @@ in rec {
};
} ./setuptools-check-hook.sh) {};
unittestCheckHook = callPackage ({ }:
makeSetupHook {
unittestCheckHook = callPackage ({ makePythonHook }:
makePythonHook {
name = "unittest-check-hook";
substitutions = {
inherit pythonCheckInterpreter;
};
} ./unittest-check-hook.sh) {};
venvShellHook = disabledIf (!isPy3k) (callPackage ({ ensureNewerSourcesForZipFilesHook }:
makeSetupHook {
venvShellHook = disabledIf (!isPy3k) (callPackage ({ makePythonHook, ensureNewerSourcesForZipFilesHook }:
makePythonHook {
name = "venv-shell-hook";
deps = [ ensureNewerSourcesForZipFilesHook ];
substitutions = {
@ -183,14 +177,18 @@ in rec {
};
} ./venv-shell-hook.sh) {});
wheelUnpackHook = callPackage ({ wheel }:
makeSetupHook {
wheelUnpackHook = callPackage ({ makePythonHook, wheel }:
makePythonHook {
name = "wheel-unpack-hook.sh";
deps = [ wheel ];
} ./wheel-unpack-hook.sh) {};
sphinxHook = callPackage ({ sphinx, installShellFiles }:
makeSetupHook {
wrapPython = callPackage ../wrap-python.nix {
inherit (pkgs.buildPackages) makeWrapper;
};
sphinxHook = callPackage ({ makePythonHook, sphinx, installShellFiles }:
makePythonHook {
name = "python${python.pythonVersion}-sphinx-hook";
deps = [ sphinx installShellFiles ];
} ./sphinx-hook.sh) {};

104
pkgs/development/interpreters/python/python-packages-base.nix Normal file
View file

@ -0,0 +1,104 @@
{ pkgs
, stdenv
, lib
, python
}:
self:
let
inherit (self) callPackage;
inherit (python.passthru) isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy310 isPy311 isPy3k isPyPy pythonAtLeast pythonOlder;
namePrefix = python.libPrefix + "-";
# Derivations built with `buildPythonPackage` can already be overriden with `override`, `overrideAttrs`, and `overrideDerivation`.
# This function introduces `overridePythonAttrs` and it overrides the call to `buildPythonPackage`.
makeOverridablePythonPackage = f: origArgs:
let
ff = f origArgs;
overrideWith = newArgs: origArgs // (if pkgs.lib.isFunction newArgs then newArgs origArgs else newArgs);
in
if builtins.isAttrs ff then (ff // {
overridePythonAttrs = newArgs: makeOverridablePythonPackage f (overrideWith newArgs);
})
else if builtins.isFunction ff then {
overridePythonAttrs = newArgs: makeOverridablePythonPackage f (overrideWith newArgs);
__functor = self: ff;
}
else ff;
buildPythonPackage = makeOverridablePythonPackage (lib.makeOverridable (callPackage ./mk-python-derivation.nix {
inherit namePrefix; # We want Python libraries to be named like e.g. "python3.6-${name}"
inherit toPythonModule; # Libraries provide modules
}));
buildPythonApplication = makeOverridablePythonPackage (lib.makeOverridable (callPackage ./mk-python-derivation.nix {
namePrefix = ""; # Python applications should not have any prefix
toPythonModule = x: x; # Application does not provide modules.
}));
# See build-setupcfg/default.nix for documentation.
buildSetupcfg = import ../../../build-support/build-setupcfg self;
fetchPypi = callPackage ./fetchpypi.nix { };
# Check whether a derivation provides a Python module.
hasPythonModule = drv: drv?pythonModule && drv.pythonModule == python;
# Get list of required Python modules given a list of derivations.
requiredPythonModules = drvs: let
modules = lib.filter hasPythonModule drvs;
in lib.unique ([python] ++ modules ++ lib.concatLists (lib.catAttrs "requiredPythonModules" modules));
# Create a PYTHONPATH from a list of derivations. This function recurses into the items to find derivations
# providing Python modules.
makePythonPath = drvs: lib.makeSearchPath python.sitePackages (requiredPythonModules drvs);
removePythonPrefix = lib.removePrefix namePrefix;
# Convert derivation to a Python module.
toPythonModule = drv:
drv.overrideAttrs( oldAttrs: {
# Use passthru in order to prevent rebuilds when possible.
passthru = (oldAttrs.passthru or {})// {
pythonModule = python;
pythonPath = [ ]; # Deprecated, for compatibility.
requiredPythonModules = requiredPythonModules drv.propagatedBuildInputs;
};
});
# Convert a Python library to an application.
toPythonApplication = drv:
drv.overrideAttrs( oldAttrs: {
passthru = (oldAttrs.passthru or {}) // {
# Remove Python prefix from name so we have a "normal" name.
# While the prefix shows up in the store path, it won't be
# used by `nix-env`.
name = removePythonPrefix oldAttrs.name;
pythonModule = false;
};
});
disabled = drv: throw "${removePythonPrefix (drv.pname or drv.name)} not supported for interpreter ${python.executable}";
disabledIf = x: drv: if x then disabled drv else drv;
in {
inherit lib pkgs stdenv;
inherit (python.passthru) isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy310 isPy311 isPy3k isPyPy pythonAtLeast pythonOlder;
inherit buildPythonPackage buildPythonApplication;
inherit fetchPypi;
inherit hasPythonModule requiredPythonModules makePythonPath disabled disabledIf;
inherit toPythonModule toPythonApplication;
inherit buildSetupcfg;
python = toPythonModule python;
# Dont take pythonPackages from "global" pkgs scope to avoid mixing python versions
pythonPackages = self;
# Remove?
recursivePthLoader = toPythonModule (callPackage ../../../development/python-modules/recursive-pth-loader { });
}

4
pkgs/development/interpreters/python/wrap-python.nix
View file

@ -1,11 +1,11 @@
{ lib
, python
, makeSetupHook
, makePythonHook
, makeWrapper }:
with lib;
makeSetupHook {
makePythonHook {
deps = makeWrapper;
substitutions.sitePackages = python.sitePackages;
substitutions.executable = python.interpreter;

141
pkgs/top-level/python-packages.nix
View file

@ -6,146 +6,9 @@
#
# For more details, please see the Python section in the Nixpkgs manual.
{ pkgs
, stdenv
, lib
, python
}:
self: super: with self; {
self:
let
inherit (self) callPackage;
inherit (python.passthru) isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy310 isPy311 isPy3k isPyPy pythonAtLeast pythonOlder;
namePrefix = python.libPrefix + "-";
bootstrapped-pip = callPackage ../development/python-modules/bootstrapped-pip { };
# Derivations built with `buildPythonPackage` can already be overriden with `override`, `overrideAttrs`, and `overrideDerivation`.
# This function introduces `overridePythonAttrs` and it overrides the call to `buildPythonPackage`.
makeOverridablePythonPackage = f: origArgs:
let
ff = f origArgs;
overrideWith = newArgs: origArgs // (if pkgs.lib.isFunction newArgs then newArgs origArgs else newArgs);
in
if builtins.isAttrs ff then (ff // {
overridePythonAttrs = newArgs: makeOverridablePythonPackage f (overrideWith newArgs);
})
else if builtins.isFunction ff then {
overridePythonAttrs = newArgs: makeOverridablePythonPackage f (overrideWith newArgs);
__functor = self: ff;
}
else ff;
buildPythonPackage = makeOverridablePythonPackage (lib.makeOverridable (callPackage ../development/interpreters/python/mk-python-derivation.nix {
inherit namePrefix; # We want Python libraries to be named like e.g. "python3.6-${name}"
inherit toPythonModule; # Libraries provide modules
}));
buildPythonApplication = makeOverridablePythonPackage (lib.makeOverridable (callPackage ../development/interpreters/python/mk-python-derivation.nix {
namePrefix = ""; # Python applications should not have any prefix
toPythonModule = x: x; # Application does not provide modules.
}));
# See build-setupcfg/default.nix for documentation.
buildSetupcfg = import ../build-support/build-setupcfg self;
fetchPypi = callPackage ../development/interpreters/python/fetchpypi.nix { };
# Check whether a derivation provides a Python module.
hasPythonModule = drv: drv?pythonModule && drv.pythonModule == python;
# Get list of required Python modules given a list of derivations.
requiredPythonModules = drvs: let
modules = lib.filter hasPythonModule drvs;
in lib.unique ([python] ++ modules ++ lib.concatLists (lib.catAttrs "requiredPythonModules" modules));
# Create a PYTHONPATH from a list of derivations. This function recurses into the items to find derivations
# providing Python modules.
makePythonPath = drvs: lib.makeSearchPath python.sitePackages (requiredPythonModules drvs);
removePythonPrefix = lib.removePrefix namePrefix;
# Convert derivation to a Python module.
toPythonModule = drv:
drv.overrideAttrs( oldAttrs: {
# Use passthru in order to prevent rebuilds when possible.
passthru = (oldAttrs.passthru or {})// {
pythonModule = python;
pythonPath = [ ]; # Deprecated, for compatibility.
requiredPythonModules = requiredPythonModules drv.propagatedBuildInputs;
};
});
# Convert a Python library to an application.
toPythonApplication = drv:
drv.overrideAttrs( oldAttrs: {
passthru = (oldAttrs.passthru or {}) // {
# Remove Python prefix from name so we have a "normal" name.
# While the prefix shows up in the store path, it won't be
# used by `nix-env`.
name = removePythonPrefix oldAttrs.name;
pythonModule = false;
};
});
disabled = drv: throw "${removePythonPrefix (drv.pname or drv.name)} not supported for interpreter ${python.executable}";
disabledIf = x: drv: if x then disabled drv else drv;
in {
inherit pkgs stdenv;
inherit (python.passthru) isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy310 isPy311 isPy3k isPyPy pythonAtLeast pythonOlder;
inherit python bootstrapped-pip buildPythonPackage buildPythonApplication;
inherit fetchPypi;
inherit hasPythonModule requiredPythonModules makePythonPath disabled disabledIf;
inherit toPythonModule toPythonApplication;
inherit buildSetupcfg;
inherit (callPackage ../development/interpreters/python/hooks { })
sphinxHook
condaInstallHook
condaUnpackHook
eggUnpackHook
eggBuildHook
eggInstallHook
flitBuildHook
pipBuildHook
pipInstallHook
pytestCheckHook
pythonCatchConflictsHook
pythonImportsCheckHook
pythonNamespacesHook
pythonOutputDistHook
pythonRecompileBytecodeHook
pythonRelaxDepsHook
pythonRemoveBinBytecodeHook
pythonRemoveTestsDirHook
setuptoolsBuildHook
setuptoolsCheckHook
unittestCheckHook
venvShellHook
wheelUnpackHook;
# helpers
# We use build packages because we are making a setup hook to be used as a
# native build input. The script itself references both the build-time
# (build) and run-time (host) python from the explicitly passed in `python`
# attribute, so the `buildPackages` doesn't effect that.
wrapPython = pkgs.buildPackages.callPackage ../development/interpreters/python/wrap-python.nix {
inherit python;
};
# Dont take pythonPackages from "global" pkgs scope to avoid mixing python versions
pythonPackages = self;
# specials
recursivePthLoader = callPackage ../development/python-modules/recursive-pth-loader { };
bootstrapped-pip = toPythonModule (callPackage ../development/python-modules/bootstrapped-pip { });
setuptools = callPackage ../development/python-modules/setuptools { };

2
pkgs/top-level/python2-packages.nix
View file

@ -7,7 +7,7 @@ self: super:
with self; with super; {
attrs = callPackage ../development/python2-modules/attrs { };
bootstrapped-pip = callPackage ../development/python2-modules/bootstrapped-pip { };
bootstrapped-pip = toPythonModule (callPackage ../development/python2-modules/bootstrapped-pip { });
boto3 = callPackage ../development/python2-modules/boto3 {};