Python: setuptools/wheel/pip now bootstrap from source

Since wheel support was introduced in 2015 we always relied on pre-built
wheels for bootstrapping. Now, we can bootstrap directly from the
sources of these packages in git.

The `bootstrapped-pip` packages is used to build `pip`, `setuptools` and `wheel`,
after which those packages are used to build everything else.

Note that when building `bootstrapped-pip` some errors are shown.
These are not important, the build actually does succeed and work as intended.

pkgs/development/python-modules/bootstrapped-pip/default.nix

@@ -1,35 +1,19 @@
 { stdenv, python, fetchPypi, makeWrapper, unzip, makeSetupHook
 , pipInstallHook
 , setuptoolsBuildHook
-
+, wheel, pip, setuptools
 }:
 
-let
+stdenv.mkDerivation rec {
-  wheel_source = fetchPypi {
-    pname = "wheel";
-    version = "0.33.6";
-    format = "wheel";
-    sha256 = "f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28";
-  };
-  setuptools_source = fetchPypi {
-    pname = "setuptools";
-    version = "41.4.0";
-    format = "wheel";
-    sha256 = "8d01f7ee4191d9fdcd9cc5796f75199deccb25b154eba82d44d6a042cf873670";
-  };
-
-in stdenv.mkDerivation rec {
   pname = "pip";
-  version = "19.3.1";
+  inherit (pip) version;
   name = "${python.libPrefix}-bootstrapped-${pname}-${version}";
 
-  src = fetchPypi {
+  srcs = [ wheel.src pip.src setuptools.src ];
-    inherit pname version;
+  sourceRoot = ".";
-    format = "wheel";
-    sha256 = "6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7";
-  };
 
   dontUseSetuptoolsBuild = true;
+  dontUsePipInstall = true;
 
   # Should be propagatedNativeBuildInputs
   propagatedBuildInputs = [
@@ -38,13 +22,6 @@ in stdenv.mkDerivation rec {
     (setuptoolsBuildHook.override{setuptools=null; wheel=null;})
   ];
 
-  unpackPhase = ''
-    mkdir -p $out/${python.sitePackages}
-    unzip -d $out/${python.sitePackages} $src
-    unzip -d $out/${python.sitePackages} ${setuptools_source}
-    unzip -d $out/${python.sitePackages} ${wheel_source}
-  '';
-
   postPatch = ''
     mkdir -p $out/bin
   '';
@@ -52,18 +29,38 @@ in stdenv.mkDerivation rec {
   nativeBuildInputs = [ makeWrapper unzip ];
   buildInputs = [ python ];
 
-  installPhase = ''
+  buildPhase = ":";
 
-    # install pip binary
+  installPhase = stdenv.lib.strings.optionalString (!stdenv.hostPlatform.isWindows) ''
-    echo '#!${python.interpreter}' > $out/bin/pip
+    export SETUPTOOLS_INSTALL_WINDOWS_SPECIFIC_FILES=0
-    echo 'import sys;from pip._internal import main' >> $out/bin/pip
+  '' + ''
-    echo 'sys.exit(main())' >> $out/bin/pip
+    # Give folders a known name
-    chmod +x $out/bin/pip
+    mv pip* pip
+    mv setuptools* setuptools
+    mv wheel* wheel
+    # Set up PYTHONPATH. The above folders need to be on PYTHONPATH
+    # $out is where we are installing to and takes precedence
+    export PYTHONPATH="$out/${python.sitePackages}:$(pwd)/pip/src:$(pwd)/setuptools:$(pwd)/setuptools/pkg_resources:$(pwd)/wheel"
 
-    # wrap binaries with PYTHONPATH
+    echo "Building setuptools wheel..."
-    for f in $out/bin/*; do
+    pushd setuptools
-      wrapProgram $f --prefix PYTHONPATH ":" $out/${python.sitePackages}/
+    ${python.pythonForBuild.interpreter} -m pip install --no-build-isolation --no-index --prefix=$out  --ignore-installed --no-dependencies --no-cache --build tmpbuild .
-    done
+    popd
+
+    echo "Building wheel wheel..."
+    pushd wheel
+    ${python.pythonForBuild.interpreter} -m pip install --no-build-isolation --no-index --prefix=$out  --ignore-installed --no-dependencies --no-cache --build tmpbuild .
+    popd
+
+    echo "Building pip wheel..."
+    pushd pip
+    ${python.pythonForBuild.interpreter} -m pip install --no-build-isolation --no-index --prefix=$out  --ignore-installed --no-dependencies --no-cache --build tmpbuild .
+    popd
   '';
 
+  meta = {
+    description = "Version of pip used for bootstrapping";
+    license = stdenv.lib.unique (pip.meta.license ++ setuptools.meta.license ++ wheel.meta.license);
+    homepage = pip.meta.homepage;
+  };
 }

pkgs/development/python-modules/pip/default.nix

@@ -2,7 +2,7 @@
 , python
 , buildPythonPackage
 , bootstrapped-pip
-, fetchPypi
+, fetchFromGitHub
 , mock
 , scripttest
 , virtualenv
@@ -17,9 +17,12 @@ buildPythonPackage rec {
   version = "19.3.1";
   format = "other";
 
-  src = fetchPypi {
+  src = fetchFromGitHub {
-    inherit pname version;
+    owner = "pypa";
-    sha256 = "21207d76c1031e517668898a6b46a9fb1501c7a4710ef5dfd6a40ad9e6757ea7";
+    repo = pname;
+    rev = version;
+    sha256 = "079gz0v37ah1l4i5iwyfb0d3mni422yv5ynnxa0wcqpnvkc7sfnw";
+    name = "${pname}-${version}-source";
   };
 
   nativeBuildInputs = [ bootstrapped-pip ];
@@ -34,7 +37,7 @@ buildPythonPackage rec {
 
   meta = {
     description = "The PyPA recommended tool for installing Python packages";
-    license = lib.licenses.mit;
+    license = with lib.licenses; [ mit ];
     homepage = https://pip.pypa.io/;
     priority = 10;
   };

pkgs/development/python-modules/setuptools/default.nix

@@ -1,6 +1,6 @@
 { stdenv
 , buildPythonPackage
-, fetchPypi
+, fetchFromGitHub
 , python
 , wrapPython
 , unzip
@@ -11,19 +11,40 @@
 , setuptoolsBuildHook
 }:
 
-buildPythonPackage rec {
+let
   pname = "setuptools";
   version = "41.4.0";
+
+  # Create an sdist of setuptools
+  sdist = stdenv.mkDerivation rec {
+    name = "${pname}-${version}-sdist.tar.gz";
+
+    src = fetchFromGitHub {
+      owner = "pypa";
+      repo = pname;
+      rev = "v${version}";
+      sha256 = "0asxfnsi56r81lm48ynqbfkmm3kvw2jwrlf2l9azn5w6xm30jvp5";
+      name = "${pname}-${version}-source";
+    };
+
+    buildPhase = ''
+      ${python.pythonForBuild.interpreter} bootstrap.py
+      ${python.pythonForBuild.interpreter} setup.py sdist --formats=gztar
+    '';
+
+    installPhase = ''
+      echo "Moving sdist..."
+      mv dist/*.tar.gz $out
+    '';
+  };
+in buildPythonPackage rec {
+  inherit pname version;
   # Because of bootstrapping we don't use the setuptoolsBuildHook that comes with format="setuptools" directly.
   # Instead, we override it to remove setuptools to avoid a circular dependency.
   # The same is done for pip and the pipInstallHook.
   format = "other";
 
-  src = fetchPypi {
+  src = sdist;
-    inherit pname version;
-    extension = "zip";
-    sha256 = "7eae782ccf36b790c21bde7d86a4f303a441cd77036b25c559a602cf5186ce4d";
-  };
 
   nativeBuildInputs = [
     bootstrapped-pip

pkgs/development/python-modules/wheel/default.nix

@@ -2,7 +2,7 @@
 , setuptools
 , pip
 , buildPythonPackage
-, fetchPypi
+, fetchFromGitHub
 , pytest
 , pytestcov
 , coverage
@@ -15,9 +15,12 @@ buildPythonPackage rec {
   version = "0.33.6";
   format = "other";
 
-  src = fetchPypi {
+  src = fetchFromGitHub {
-    inherit pname version;
+    owner = "pypa";
-    sha256 = "10c9da68765315ed98850f8e048347c3eb06dd81822dc2ab1d4fde9dc9702646";
+    repo = pname;
+    rev = version;
+    sha256 = "1bg4bxazsjxp621ymaykd8l75k7rvcvwawlipmjk7nsrl72l4p0s";
+    name = "${pname}-${version}-source";
   };
 
   checkInputs = [ pytest pytestcov coverage ];