Merge pull request #20893 from FRidh/fixed3

Python: use fixed-point combinator, add pkgs to interpreter

doc/languages-frameworks/python.md

@@ -434,6 +434,7 @@ Each interpreter has the following attributes:
 - `withPackages`. Simpler interface to `buildEnv`. See section *python.withPackages function* for usage and documentation.
 - `sitePackages`. Alias for `lib/${libPrefix}/site-packages`.
 - `executable`. Name of the interpreter executable, e.g. `python3.4`.
+- `pkgs`. Set of Python packages for that specific interpreter. The package set can be modified by overriding the interpreter and passing `packageOverrides`.
 
 ### Building packages and applications
 
@@ -699,59 +700,55 @@ should also be done when packaging `A`.
 
 ### How to override a Python package?
 
-Recursively updating a package can be done with `pkgs.overridePackages` as explained in the Nixpkgs manual.
-Python attribute sets are created for each interpreter version. We will therefore override the attribute set for the interpreter version we're interested.
-In the following example we change the name of the package `pandas` to `foo`.
-```
-newpkgs = pkgs.overridePackages(self: super: rec {
-  python35Packages = (super.python35Packages.override { self = python35Packages;})
-    // { pandas = super.python35Packages.pandas.override  {name = "foo";};
-  };
-});
-```
-This can be tested with
-```
+We can override the interpreter and pass `packageOverrides`.
+In the following example we rename the `pandas` package and build it.
+```nix
 with import <nixpkgs> {};
 
-(let
+let
+  python = let
+    packageOverrides = self: super: {
+      pandas = super.pandas.override {name="foo";};
+    };
+  in pkgs.python35.override {inherit packageOverrides;};
 
-newpkgs = pkgs.overridePackages(self: super: rec {
-  python35Packages = (super.python35Packages.override { self = python35Packages;})
-    // { pandas = super.python35Packages.pandas.override  {name = "foo";};
-  };
-});
-in newpkgs.python35.withPackages (ps: [ps.blaze])
-).env
-```
-A typical use case is to switch to another version of a certain package. For example, in the Nixpkgs repository we have multiple versions of `django` and `scipy`.
-In the following example we use a different version of `scipy`. All packages in `newpkgs` will now use the updated `scipy` version.
+in python.pkgs.pandas
 ```
+Using `nix-build` on this expression will build the package `pandas`
+but with the new name `foo`.
+
+All packages in the package set will use the renamed package.
+A typical use case is to switch to another version of a certain package.
+For example, in the Nixpkgs repository we have multiple versions of `django` and `scipy`.
+In the following example we use a different version of `scipy` and create an environment that uses it.
+All packages in the Python package set will now use the updated `scipy` version.
+
+```nix
 with import <nixpkgs> {};
 
-(let
-
-newpkgs = pkgs.overridePackages(self: super: rec {
-  python35Packages = super.python35Packages.override {
-    self = python35Packages // { scipy = python35Packages.scipy_0_17;};
+(
+let
+  packageOverrides = self: super: {
+    scipy = super.scipy_0_17;
   };
-});
-in newpkgs.python35.withPackages (ps: [ps.blaze])
+in (pkgs.python35.override {inherit packageOverrides;}).withPackages (ps: [ps.blaze])
 ).env
 ```
-The requested package `blaze` depends upon `pandas` which itself depends on `scipy`.
+The requested package `blaze` depends on `pandas` which itself depends on `scipy`.
 
-A similar example but now using `django`
+If you want the whole of Nixpkgs to use your modifications, then you can use `pkgs.overridePackages`
+as explained in this manual. In the following example we build a `inkscape` using a different version of `numpy`.
 ```
-with import <nixpkgs> {};
-
-(let
-
-newpkgs = pkgs.overridePackages(self: super: rec {
-  python27Packages = (super.python27Packages.override {self = python27Packages;})
-    // { django = super.python27Packages.django_1_9; };
-});
-in newpkgs.python27.withPackages (ps: [ps.django_guardian ])
-).env
+let
+  pkgs = import <nixpkgs> {};
+  newpkgs = pkgs.overridePackages ( pkgsself: pkgssuper: {
+    python27 = let
+      packageOverrides = self: super: {
+        numpy = super.numpy_1_10;
+      };
+    in pkgssuper.python27.override {inherit packageOverrides;};
+  } );
+in newpkgs.inkscape
 ```
 
 ### `python setup.py bdist_wheel` cannot create .whl
@@ -772,9 +769,9 @@ or the current time:
 nix-shell --run "SOURCE_DATE_EPOCH=$(date +%s) python3 setup.py bdist_wheel"
 ```
 or unset:
-"""
+```
 nix-shell --run "unset SOURCE_DATE_EPOCH; python3 setup.py bdist_wheel"
-"""
+```
 
 ### `install_data` / `data_files` problems
 

pkgs/development/interpreters/python/cpython/2.6/default.nix

@@ -1,6 +1,8 @@
 { stdenv, fetchurl, zlib ? null, zlibSupport ? true, bzip2, includeModules ? false
 , sqlite, tcl, tk, xlibsWrapper, openssl, readline, db, ncurses, gdbm, self, callPackage
-, python26Packages }:
+# For the Python package set
+, pkgs, packageOverrides ? (self: super: {})
+}:
 
 assert zlibSupport -> zlib != null;
 
@@ -100,13 +102,16 @@ let
         ${ optionalString includeModules "$out/bin/python ./setup.py build_ext"}
       '';
 
-    passthru = rec {
+    passthru = let
+      pythonPackages = callPackage ../../../../../top-level/python-packages.nix {python=self; overrides=packageOverrides;};
+    in rec {
       inherit libPrefix;
       inherit zlibSupport;
       isPy2 = true;
       isPy26 = true;
       buildEnv = callPackage ../../wrapper.nix { python = self; };
-      withPackages = import ../../with-packages.nix { inherit buildEnv; pythonPackages = python26Packages; };
+      withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;};
+      pkgs = pythonPackages;
       executable = libPrefix;
       sitePackages = "lib/${libPrefix}/site-packages";
       interpreter = "${self}/bin/${executable}";

pkgs/development/interpreters/python/cpython/2.7/default.nix

@@ -10,12 +10,13 @@
 , zlib
 , callPackage
 , self
-, python27Packages
 , gettext
 , db
 , expat
 , libffi
 , CF, configd, coreutils
+# For the Python package set
+, pkgs, packageOverrides ? (self: super: {})
 }:
 
 assert x11Support -> tcl != null
@@ -180,11 +181,14 @@ in stdenv.mkDerivation {
         rm "$out"/lib/python*/plat-*/regen # refers to glibc.dev
       '';
 
-    passthru = rec {
+    passthru = let
+      pythonPackages = callPackage ../../../../../top-level/python-packages.nix {python=self; overrides=packageOverrides;};
+    in rec {
       inherit libPrefix sitePackages x11Support hasDistutilsCxxPatch;
       executable = libPrefix;
       buildEnv = callPackage ../../wrapper.nix { python = self; };
-      withPackages = import ../../with-packages.nix { inherit buildEnv; pythonPackages = python27Packages; };
+      withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;};
+      pkgs = pythonPackages;
       isPy2 = true;
       isPy27 = true;
       interpreter = "${self}/bin/${executable}";

pkgs/development/interpreters/python/cpython/3.3/default.nix

@@ -10,8 +10,9 @@
 , zlib
 , callPackage
 , self
-, python33Packages
 , CF, configd
+# For the Python package set
+, pkgs, packageOverrides ? (self: super: {})
 }:
 
 assert x11Support -> tcl != null
@@ -102,11 +103,14 @@ in stdenv.mkDerivation {
 
   '';
 
-  passthru = rec {
+  passthru = let
+    pythonPackages = callPackage ../../../../../top-level/python-packages.nix {python=self; overrides=packageOverrides;};
+  in rec {
     inherit libPrefix sitePackages x11Support;
     executable = "${libPrefix}m";
     buildEnv = callPackage ../../wrapper.nix { python = self; };
-    withPackages = import ../../with-packages.nix { inherit buildEnv; pythonPackages = python33Packages; };
+    withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;};
+    pkgs = pythonPackages;
     isPy3 = true;
     isPy33 = true;
     is_py3k = true;  # deprecated

pkgs/development/interpreters/python/cpython/3.4/default.nix

@@ -10,8 +10,9 @@
 , zlib
 , callPackage
 , self
-, python34Packages
 , CF, configd
+# For the Python package set
+, pkgs, packageOverrides ? (self: super: {})
 }:
 
 assert x11Support -> tcl != null
@@ -111,11 +112,14 @@ in stdenv.mkDerivation {
 
   '';
 
-  passthru = rec {
+  passthru = let
+    pythonPackages = callPackage ../../../../../top-level/python-packages.nix {python=self; overrides=packageOverrides;};
+  in rec {
     inherit libPrefix sitePackages x11Support;
     executable = "${libPrefix}m";
     buildEnv = callPackage ../../wrapper.nix { python = self; };
-    withPackages = import ../../with-packages.nix { inherit buildEnv; pythonPackages = python34Packages; };
+    withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;};
+    pkgs = pythonPackages;
     isPy3 = true;
     isPy34 = true;
     is_py3k = true;  # deprecated

pkgs/development/interpreters/python/cpython/3.5/default.nix

@@ -10,8 +10,9 @@
 , zlib
 , callPackage
 , self
-, python35Packages
 , CF, configd
+# For the Python package set
+, pkgs, packageOverrides ? (self: super: {})
 }:
 
 assert x11Support -> tcl != null
@@ -110,11 +111,14 @@ in stdenv.mkDerivation {
     rm $out/lib/python${majorVersion}/__pycache__/_sysconfigdata.cpython*
   '';
 
-  passthru = rec {
+  passthru = let
+    pythonPackages = callPackage ../../../../../top-level/python-packages.nix {python=self; overrides=packageOverrides;};
+  in rec {
     inherit libPrefix sitePackages x11Support;
     executable = "${libPrefix}m";
     buildEnv = callPackage ../../wrapper.nix { python = self; };
-    withPackages = import ../../with-packages.nix { inherit buildEnv; pythonPackages = python35Packages; };
+    withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;};
+    pkgs = pythonPackages;
     isPy3 = true;
     isPy35 = true;
     interpreter = "${self}/bin/${executable}";

pkgs/development/interpreters/python/cpython/3.6/default.nix

@@ -11,8 +11,9 @@
 , zlib
 , callPackage
 , self
-, python36Packages
 , CF, configd
+# For the Python package set
+, pkgs, packageOverrides ? (self: super: {})
 }:
 
 assert x11Support -> tcl != null
@@ -99,11 +100,14 @@ in stdenv.mkDerivation {
     ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
   '';
 
-  passthru = rec {
+  passthru = let
+    pythonPackages = callPackage ../../../../../top-level/python-packages.nix {python=self; overrides=packageOverrides;};
+  in rec {
     inherit libPrefix sitePackages x11Support;
     executable = "${libPrefix}m";
     buildEnv = callPackage ../../wrapper.nix { python = self; };
-    withPackages = import ../../with-packages.nix { inherit buildEnv; pythonPackages = python36Packages; };
+    withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;};
+    pkgs = pythonPackages;
     isPy3 = true;
     isPy35 = true;
     is_py3k = true;  # deprecated

pkgs/development/interpreters/python/pypy/2.7/default.nix

@@ -1,6 +1,9 @@
 { stdenv, fetchurl, zlib ? null, zlibSupport ? true, bzip2, pkgconfig, libffi
 , sqlite, openssl, ncurses, python, expat, tcl, tk, xlibsWrapper, libX11
-, makeWrapper, callPackage, self, pypyPackages, gdbm, db }:
+, makeWrapper, callPackage, self, gdbm, db
+# For the Python package set
+, pkgs, packageOverrides ? (self: super: {})
+}:
 
 assert zlibSupport -> zlib != null;
 
@@ -120,14 +123,17 @@ let
         echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py
     '';
 
-    passthru = rec {
+    passthru = let
+      pythonPackages = callPackage ../../../../../top-level/python-packages.nix {python=self; overrides=packageOverrides;};
+    in rec {
       inherit zlibSupport libPrefix;
       executable = "pypy";
       isPypy = true;
       buildEnv = callPackage ../../wrapper.nix { python = self; };
       interpreter = "${self}/bin/${executable}";
       sitePackages = "site-packages";
-      withPackages = import ../../with-packages.nix { inherit buildEnv; pythonPackages = pypyPackages; };
+      withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;};
+      pkgs = pythonPackages;
     };
 
     enableParallelBuilding = true;  # almost no parallelization without STM

pkgs/top-level/all-packages.nix

@@ -9898,40 +9898,19 @@ in
   # Therefore we do not recurse into attributes here, in contrast to
   # python27Packages. `nix-env -iA python26Packages.nose` works
   # regardless.
-  python26Packages = callPackage ./python-packages.nix {
-    python = python26;
-    self = python26Packages;
-  };
+  python26Packages = python26.pkgs;
 
-  python27Packages = lib.hiPrioSet (recurseIntoAttrs (callPackage ./python-packages.nix {
-    python = python27;
-    self = python27Packages;
-  }));
+  python27Packages = lib.hiPrioSet (recurseIntoAttrs python27.pkgs);
 
-  python33Packages = callPackage ./python-packages.nix {
-    python = python33;
-    self = python33Packages;
-  };
+  python33Packages = python33.pkgs;
 
-  python34Packages = callPackage ./python-packages.nix {
-    python = python34;
-    self = python34Packages;
-  };
+  python34Packages = python34.pkgs;
 
-  python35Packages = recurseIntoAttrs (callPackage ./python-packages.nix {
-    python = python35;
-    self = python35Packages;
-  });
+  python35Packages = recurseIntoAttrs python35.pkgs;
 
-  python36Packages = (callPackage ./python-packages.nix {
-    python = python36;
-    self = python36Packages;
-  });
+  python36Packages = python36.pkgs;
 
-  pypyPackages = callPackage ./python-packages.nix {
-    python = pypy;
-    self = pypyPackages;
-  };
+  pypyPackages = pypy.pkgs;
 
   ### DEVELOPMENT / R MODULES
 

pkgs/top-level/python-packages.nix

@@ -1,7 +1,14 @@
-{ pkgs, stdenv, python, self }:
+{ pkgs
+, stdenv
+, python
+, overrides ? (self: super: {})
+}:
 
 with pkgs.lib;
 
+let
+  packages = ( self:
+
 let
   pythonAtLeast = versionAtLeast python.pythonVersion;
   pythonOlder = versionOlder python.pythonVersion;
@@ -31323,4 +31330,7 @@ in {
   zeitgeist = if isPy3k then throw "zeitgeist not supported for interpreter ${python.executable}" else
     (pkgs.zeitgeist.override{python2Packages=self;}).py;
 
-}
+
+});
+
+in fix' (extends overrides packages)