From c3ed3914bbde9c95c52d41a6732068e62b4ee65a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benno=20F=C3=BCnfst=C3=BCck?= Date: Sun, 29 May 2016 15:12:23 +0200 Subject: [PATCH 1/4] python: add python.withPackages function Fixes #15801 --- pkgs/development/interpreters/python/2.6/default.nix | 4 +++- pkgs/development/interpreters/python/2.7/default.nix | 3 ++- pkgs/development/interpreters/python/3.3/default.nix | 2 ++ pkgs/development/interpreters/python/3.4/default.nix | 2 ++ pkgs/development/interpreters/python/3.5/default.nix | 2 ++ pkgs/development/interpreters/python/with-packages.nix | 3 +++ 6 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 pkgs/development/interpreters/python/with-packages.nix diff --git a/pkgs/development/interpreters/python/2.6/default.nix b/pkgs/development/interpreters/python/2.6/default.nix index 548b7bcecbc7..726e2aa6aca1 100644 --- a/pkgs/development/interpreters/python/2.6/default.nix +++ b/pkgs/development/interpreters/python/2.6/default.nix @@ -1,5 +1,6 @@ { stdenv, fetchurl, zlib ? null, zlibSupport ? true, bzip2, includeModules ? false -, sqlite, tcl, tk, xlibsWrapper, openssl, readline, db, ncurses, gdbm, self, callPackage }: +, sqlite, tcl, tk, xlibsWrapper, openssl, readline, db, ncurses, gdbm, self, callPackage +, python26Packages }: assert zlibSupport -> zlib != null; @@ -97,6 +98,7 @@ let isPy2 = true; isPy26 = true; buildEnv = callPackage ../wrapper.nix { python = self; }; + withPackages = import ../with-packages.nix { inherit buildEnv; pythonPackages = python26Packages; }; libPrefix = "python${majorVersion}"; executable = libPrefix; sitePackages = "lib/${libPrefix}/site-packages"; diff --git a/pkgs/development/interpreters/python/2.7/default.nix b/pkgs/development/interpreters/python/2.7/default.nix index 2e94cb6874e0..a72377a47708 100644 --- a/pkgs/development/interpreters/python/2.7/default.nix +++ b/pkgs/development/interpreters/python/2.7/default.nix @@ -1,4 +1,4 @@ -{ stdenv, fetchurl, self, callPackage +{ stdenv, fetchurl, self, callPackage, python27Packages , bzip2, openssl, gettext , includeModules ? false @@ -151,6 +151,7 @@ let isPy2 = true; isPy27 = true; buildEnv = callPackage ../wrapper.nix { python = self; }; + withPackages = import ../with-packages.nix { inherit buildEnv; pythonPackages = python27Packages; }; libPrefix = "python${majorVersion}"; executable = libPrefix; sitePackages = "lib/${libPrefix}/site-packages"; diff --git a/pkgs/development/interpreters/python/3.3/default.nix b/pkgs/development/interpreters/python/3.3/default.nix index 3c4580a061f1..8c16995d5cc4 100644 --- a/pkgs/development/interpreters/python/3.3/default.nix +++ b/pkgs/development/interpreters/python/3.3/default.nix @@ -12,6 +12,7 @@ , zlib , callPackage , self +, python33Packages }: assert readline != null -> ncurses != null; @@ -81,6 +82,7 @@ stdenv.mkDerivation { libPrefix = "python${majorVersion}"; executable = "python3.3m"; buildEnv = callPackage ../wrapper.nix { python = self; }; + withPackages = import ../with-packages.nix { inherit buildEnv; pythonPackages = python33Packages; }; isPy3 = true; isPy33 = true; is_py3k = true; # deprecated diff --git a/pkgs/development/interpreters/python/3.4/default.nix b/pkgs/development/interpreters/python/3.4/default.nix index b36eda67867b..197ad6fc95bc 100644 --- a/pkgs/development/interpreters/python/3.4/default.nix +++ b/pkgs/development/interpreters/python/3.4/default.nix @@ -12,6 +12,7 @@ , zlib , callPackage , self +, python34Packages , CF, configd }: @@ -104,6 +105,7 @@ stdenv.mkDerivation { libPrefix = "python${majorVersion}"; executable = "python3.4m"; buildEnv = callPackage ../wrapper.nix { python = self; }; + withPackages = import ../with-packages.nix { inherit buildEnv; pythonPackages = python34Packages; }; isPy3 = true; isPy34 = true; is_py3k = true; # deprecated diff --git a/pkgs/development/interpreters/python/3.5/default.nix b/pkgs/development/interpreters/python/3.5/default.nix index 087b5988e26a..762ef1ab8be6 100644 --- a/pkgs/development/interpreters/python/3.5/default.nix +++ b/pkgs/development/interpreters/python/3.5/default.nix @@ -12,6 +12,7 @@ , zlib , callPackage , self +, python35Packages , CF, configd }: @@ -104,6 +105,7 @@ stdenv.mkDerivation { libPrefix = "python${majorVersion}"; executable = "python${majorVersion}m"; buildEnv = callPackage ../wrapper.nix { python = self; }; + withPackages = import ../with-packages.nix { inherit buildEnv; pythonPackages = python35Packages; }; isPy3 = true; isPy35 = true; is_py3k = true; # deprecated diff --git a/pkgs/development/interpreters/python/with-packages.nix b/pkgs/development/interpreters/python/with-packages.nix new file mode 100644 index 000000000000..e1de0b2ee4ca --- /dev/null +++ b/pkgs/development/interpreters/python/with-packages.nix @@ -0,0 +1,3 @@ +{ buildEnv, pythonPackages }: + +f: let packages = f pythonPackages; in buildEnv.override { extraLibs = packages; } From 3bdf167619979fa23391ab5624c735521ec53179 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benno=20F=C3=BCnfst=C3=BCck?= Date: Sun, 29 May 2016 16:08:27 +0200 Subject: [PATCH 2/4] doc/python: document python.withPackages --- doc/languages-frameworks/python.md | 31 ++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/doc/languages-frameworks/python.md b/doc/languages-frameworks/python.md index a04ca75a2fbb..ae5fa702da6c 100644 --- a/doc/languages-frameworks/python.md +++ b/doc/languages-frameworks/python.md @@ -581,6 +581,37 @@ specified packages in its path. * `postBuild`: Shell command executed after the build of environment. * `ignoreCollisions`: Ignore file collisions inside the environment (default is `false`). +#### python.withPackages function + +The `python.withPackages` function provides a simpler interface to the `python.buildEnv` functionality. +It takes a function as an argument that is passed the set of python packages and returns the list +of the packages to be included in the environment. Using the `withPackages` function, the previous +example for the Pyramid Web Framework environment can be written like this: + + with import {}; + + python.withPackages (ps: [ps.pyramid]) + +`withPackages` passes the correct package set for the specific interpreter version as an +argument to the function. In the above example, `ps` equals `pythonPackages`. +But you can also easily switch to using python3: + + with import {}; + + python3.withPackages (ps: [ps.pyramid]) + +Now, `ps` is set to `python3Packages`, matching the version of the interpreter. + +As `python.withPackages` simply uses `python.buildEnv` under the hood, it also supports the `env` +attribute. The `shell.nix` file from the previous section can thus be also written like this: + + with import {}; + + (python33.withPackages (ps: [ps.numpy ps.requests])).env + +In contrast to `python.buildEnv`, `python.withPackages` does not support the more advanced options +such as `ignoreCollisions = true` or `postBuild`. If you need them, you have to use `python.buildEnv`. + ### Development mode Development or editable mode is supported. To develop Python packages From bad156a0d57c79edcd4d4a135c5586b9f4aaa256 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benno=20F=C3=BCnfst=C3=BCck?= Date: Sun, 29 May 2016 16:30:29 +0200 Subject: [PATCH 3/4] doc/python: replace buildEnv usage by withPackages --- doc/languages-frameworks/python.md | 37 +++++++++++++----------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/doc/languages-frameworks/python.md b/doc/languages-frameworks/python.md index ae5fa702da6c..91c15ccd9e42 100644 --- a/doc/languages-frameworks/python.md +++ b/doc/languages-frameworks/python.md @@ -78,18 +78,16 @@ containing ```nix with import {}; -(pkgs.python35.buildEnv.override { - extraLibs = with pkgs.python35Packages; [ numpy toolz ]; -}).env +(pkgs.python35.withPackages (ps: [ps.numpy ps.toolz])).env ``` executing `nix-shell` gives you again a Nix shell from which you can run Python. What's happening here? 1. We begin with importing the Nix Packages collections. `import ` import the `` function, `{}` calls it and the `with` statement brings all attributes of `nixpkgs` in the local scope. Therefore we can now use `pkgs`. -2. Then we create a Python 3.5 environment with `pkgs.buildEnv`. Because we want to use it with a custom set of Python packages, we override it. -3. The `extraLibs` argument of the original `buildEnv` function can be used to specify which packages should be included. We want `numpy` and `toolz`. Again, we use the `with` statement to bring a set of attributes into the local scope. -4. And finally, for in interactive use we return the environment. +2. Then we create a Python 3.5 environment with the `withPackages` function. +3. The `withPackages` function expects us to provide a function as an argument that takes the set of all python packages and returns a list of packages to include in the environment. Here, we select the packages `numpy` and `toolz` from the package set. +4. And finally, for in interactive use we return the environment by using the `env` attribute. ### Developing with Python @@ -187,10 +185,7 @@ with import {}; }; }; - in pkgs.python35.buildEnv.override rec { - - extraLibs = [ pkgs.python35Packages.numpy toolz ]; -} + in pkgs.python35.withPackages (ps: [ps.numpy toolz]) ).env ``` @@ -199,8 +194,11 @@ locally defined package as well as `numpy` which is build according to the definition in Nixpkgs. What did we do here? Well, we took the Nix expression that we used earlier to build a Python environment, and said that we wanted to include our own version of `toolz`. To introduce our own package in the scope of -`buildEnv.override` we used a +`withPackages` we used a [`let`](http://nixos.org/nix/manual/#sec-constructs) expression. +You can see that we used `ps.numpy` to select numpy from the nixpkgs package set (`ps`). +But we do not take `toolz` from the nixpkgs package set this time. +Instead, `toolz` will resolve to our local definition that we introduced with `let`. ### Handling dependencies @@ -359,7 +357,7 @@ own packages. The important functions here are `import` and `callPackage`. ### Including a derivation using `callPackage` -Earlier we created a Python environment using `buildEnv`, and included the +Earlier we created a Python environment using `withPackages`, and included the `toolz` package via a `let` expression. Let's split the package definition from the environment definition. @@ -394,9 +392,7 @@ with import {}; ( let toolz = pkgs.callPackage ~/path/to/toolz/release.nix { pkgs=pkgs; buildPythonPackage=pkgs.python35Packages.buildPythonPackage; }; - in pkgs.python35.buildEnv.override rec { - extraLibs = [ pkgs.python35Packages.numpy toolz ]; -} + in pkgs.python35.withPackages (ps: [ ps.numpy toolz ]) ).env ``` @@ -450,6 +446,7 @@ Each interpreter has the following attributes: - `libPrefix`. Name of the folder in `${python}/lib/` for corresponding interpreter. - `interpreter`. Alias for `${python}/bin/${executable}`. - `buildEnv`. Function to build python interpreter environments with extra packages bundled together. See section *python.buildEnv function* for usage and documentation. +- `withPackages`. Simpler interface to `buildEnv`. See section *python.withPackages function* for usage and documentation. - `sitePackages`. Alias for `lib/${libPrefix}/site-packages`. - `executable`. Name of the interpreter executable, ie `python3.4`. @@ -680,9 +677,8 @@ newpkgs = pkgs.overridePackages(self: super: rec { self = python35Packages // { pandas = python35Packages.pandas.override{name="foo";};}; }; }); -in newpkgs.python35.buildEnv.override{ - extraLibs = [newpkgs.python35Packages.blaze ]; -}).env +in newpkgs.python35.withPackages (ps: [ps.blaze]) +).env ``` A typical use case is to switch to another version of a certain package. For example, in the Nixpkgs repository we have multiple versions of `django` and `scipy`. In the following example we use a different version of `scipy`. All packages in `newpkgs` will now use the updated `scipy` version. @@ -696,9 +692,8 @@ newpkgs = pkgs.overridePackages(self: super: rec { self = python35Packages // { scipy = python35Packages.scipy_0_16;}; }; }); -in pkgs.python35.buildEnv.override{ - extraLibs = [newpkgs.python35Packages.blaze ]; -}).env +in newpkgs.python35.withPackages (ps: [ps.blaze]) +).env ``` The requested package `blaze` depends upon `pandas` which itself depends on `scipy`. From 5e0acb90d6a3eb43e5b150ce9ed86b68ac83c708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benno=20F=C3=BCnfst=C3=BCck?= Date: Sun, 29 May 2016 16:44:54 +0200 Subject: [PATCH 4/4] doc/python: fix conversion errors in example code --- doc/languages-frameworks/python.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/languages-frameworks/python.md b/doc/languages-frameworks/python.md index 91c15ccd9e42..50acc7f28f78 100644 --- a/doc/languages-frameworks/python.md +++ b/doc/languages-frameworks/python.md @@ -545,7 +545,7 @@ Python environments can be created using the low-level `pkgs.buildEnv` function. This example shows how to create an environment that has the Pyramid Web Framework. Saving the following as `default.nix` - with import {}; + with import {}; python.buildEnv.override { extraLibs = [ pkgs.pythonPackages.pyramid ]; @@ -562,7 +562,7 @@ You can also use the `env` attribute to create local environments with needed packages installed. This is somewhat comparable to `virtualenv`. For example, running `nix-shell` with the following `shell.nix` - with import {}; + with import {}; (python3.buildEnv.override { extraLibs = with python3Packages; [ numpy requests ]; @@ -585,7 +585,7 @@ It takes a function as an argument that is passed the set of python packages and of the packages to be included in the environment. Using the `withPackages` function, the previous example for the Pyramid Web Framework environment can be written like this: - with import {}; + with import {}; python.withPackages (ps: [ps.pyramid]) @@ -593,7 +593,7 @@ example for the Pyramid Web Framework environment can be written like this: argument to the function. In the above example, `ps` equals `pythonPackages`. But you can also easily switch to using python3: - with import {}; + with import {}; python3.withPackages (ps: [ps.pyramid]) @@ -602,7 +602,7 @@ Now, `ps` is set to `python3Packages`, matching the version of the interpreter. As `python.withPackages` simply uses `python.buildEnv` under the hood, it also supports the `env` attribute. The `shell.nix` file from the previous section can thus be also written like this: - with import {}; + with import {}; (python33.withPackages (ps: [ps.numpy ps.requests])).env @@ -619,7 +619,7 @@ Warning: `shellPhase` is executed only if `setup.py` exists. Given a `default.nix`: - with import {}; + with import {}; buildPythonPackage { name = "myproject";