cpython: fix finding headers when cross-compiling extension modules

pkgs/development/interpreters/python/cpython/3.6/fix-finding-headers-when-cross-compiling.patch

@@ -0,0 +1,54 @@
+From 45dfbbb4f5b67ab83e4365564ea569334e979f8e Mon Sep 17 00:00:00 2001
+From: Ben Wolsieffer <benwolsieffer@gmail.com>
+Date: Fri, 25 Sep 2020 16:49:16 -0400
+Subject: [PATCH] Fix finding headers when cross compiling
+
+When cross-compiling third-party extensions, get_python_inc() may be called to
+return the path to Python's headers. However, it uses the sys.prefix or
+sys.exec_prefix of the build Python, which returns incorrect paths when
+cross-compiling (paths pointing to build system headers).
+
+To fix this, we use the INCLUDEPY and CONFINCLUDEPY conf variables, which can
+be configured to point at host Python by setting _PYTHON_SYSCONFIGDATA_NAME.
+The existing behavior is maintained on non-POSIX platforms or if a prefix is
+manually specified.
+---
+ Lib/distutils/sysconfig.py | 14 ++++++++++----
+ 1 file changed, 10 insertions(+), 4 deletions(-)
+
+diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py
+index 2bcd1dd288..567375e488 100644
+--- a/Lib/distutils/sysconfig.py
++++ b/Lib/distutils/sysconfig.py
+@@ -84,8 +84,6 @@ def get_python_inc(plat_specific=0, prefix=None):
+     If 'prefix' is supplied, use it instead of sys.base_prefix or
+     sys.base_exec_prefix -- i.e., ignore 'plat_specific'.
+     """
+-    if prefix is None:
+-        prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX
+     if os.name == "posix":
+         if python_build:
+             # Assume the executable is in the build directory.  The
+@@ -98,9 +96,17 @@ def get_python_inc(plat_specific=0, prefix=None):
+             else:
+                 incdir = os.path.join(get_config_var('srcdir'), 'Include')
+                 return os.path.normpath(incdir)
+-        python_dir = 'python' + get_python_version() + build_flags
+-        return os.path.join(prefix, "include", python_dir)
++        if prefix is None:
++          if plat_specific:
++            return get_config_var('CONFINCLUDEPY')
++          else:
++            return get_config_var('INCLUDEPY')
++        else:
++          python_dir = 'python' + get_python_version() + build_flags
++          return os.path.join(prefix, "include", python_dir)
+     elif os.name == "nt":
++        if prefix is None:
++          prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX
+         return os.path.join(prefix, "include")
+     else:
+         raise DistutilsPlatformError(
+-- 
+2.28.0
+

pkgs/development/interpreters/python/cpython/3.7/fix-finding-headers-when-cross-compiling.patch

@@ -0,0 +1,54 @@
+From debccd4be0a8d619770f63622d9de1b451dd02ac Mon Sep 17 00:00:00 2001
+From: Ben Wolsieffer <benwolsieffer@gmail.com>
+Date: Fri, 25 Sep 2020 16:49:16 -0400
+Subject: [PATCH] Fix finding headers when cross compiling
+
+When cross-compiling third-party extensions, get_python_inc() may be called to
+return the path to Python's headers. However, it uses the sys.prefix or
+sys.exec_prefix of the build Python, which returns incorrect paths when
+cross-compiling (paths pointing to build system headers).
+
+To fix this, we use the INCLUDEPY and CONFINCLUDEPY conf variables, which can
+be configured to point at host Python by setting _PYTHON_SYSCONFIGDATA_NAME.
+The existing behavior is maintained on non-POSIX platforms or if a prefix is
+manually specified.
+---
+ Lib/distutils/sysconfig.py | 14 ++++++++++----
+ 1 file changed, 10 insertions(+), 4 deletions(-)
+
+diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py
+index 37feae5df7..6d4ad06696 100644
+--- a/Lib/distutils/sysconfig.py
++++ b/Lib/distutils/sysconfig.py
+@@ -95,8 +95,6 @@ def get_python_inc(plat_specific=0, prefix=None):
+     If 'prefix' is supplied, use it instead of sys.base_prefix or
+     sys.base_exec_prefix -- i.e., ignore 'plat_specific'.
+     """
+-    if prefix is None:
+-        prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX
+     if os.name == "posix":
+         if python_build:
+             # Assume the executable is in the build directory.  The
+@@ -109,9 +107,17 @@ def get_python_inc(plat_specific=0, prefix=None):
+             else:
+                 incdir = os.path.join(get_config_var('srcdir'), 'Include')
+                 return os.path.normpath(incdir)
+-        python_dir = 'python' + get_python_version() + build_flags
+-        return os.path.join(prefix, "include", python_dir)
++        if prefix is None:
++          if plat_specific:
++            return get_config_var('CONFINCLUDEPY')
++          else:
++            return get_config_var('INCLUDEPY')
++        else:
++          python_dir = 'python' + get_python_version() + build_flags
++          return os.path.join(prefix, "include", python_dir)
+     elif os.name == "nt":
++        if prefix is None:
++          prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX
+         if python_build:
+             # Include both the include and PC dir to ensure we can find
+             # pyconfig.h
+-- 
+2.28.0
+

pkgs/development/interpreters/python/cpython/default.nix

@@ -101,6 +101,44 @@ let
     "$out/bin/python"
   else pythonForBuild.interpreter;
 
+  # The CPython interpreter contains a _sysconfigdata_<platform specific suffix>
+  # module that is imported by the sysconfig and distutils.sysconfig modules.
+  # The sysconfigdata module is generated at build time and contains settings
+  # required for building Python extension modules, such as include paths and
+  # other compiler flags. By default, the sysconfigdata module is loaded from
+  # the currently running interpreter (ie. the build platform interpreter), but
+  # when cross-compiling we want to load it from the host platform interpreter.
+  # This can be done using the _PYTHON_SYSCONFIGDATA_NAME environment variable.
+  # The _PYTHON_HOST_PLATFORM variable also needs to be set to get the correct
+  # platform suffix on extension modules. The correct values for these variables
+  # are not documented, and must be derived from the configure script (see links
+  # below).
+  sysconfigdataHook = with stdenv.hostPlatform; let
+    # https://github.com/python/cpython/blob/e488e300f5c01289c10906c2e53a8e43d6de32d8/configure.ac#L428
+    pythonHostPlatform = "${parsed.kernel.name}-${parsed.cpu.name}";
+
+    # https://github.com/python/cpython/blob/e488e300f5c01289c10906c2e53a8e43d6de32d8/configure.ac#L724
+    multiarchCpu =
+      if isAarch32 then
+        if parsed.cpu.significantByte.name == "littleEndian" then "arm" else "armeb"
+      else parsed.cpu.name;
+    multiarch =
+      if isDarwin then "darwin"
+      else "${multiarchCpu}-${parsed.kernel.name}-${parsed.abi.name}";
+
+    # https://github.com/python/cpython/blob/e488e300f5c01289c10906c2e53a8e43d6de32d8/configure.ac#L78
+    pythonSysconfigdataName = "_sysconfigdata__${parsed.kernel.name}_${multiarch}";
+  in ''
+    sysconfigdataHook() {
+      if [ "$1" = '${placeholder "out"}' ]; then
+        export _PYTHON_HOST_PLATFORM='${pythonHostPlatform}'
+        export _PYTHON_SYSCONFIGDATA_NAME='${pythonSysconfigdataName}'
+      fi
+    }
+
+    addEnvHooks "$hostOffset" sysconfigdataHook
+  '';
+
 in with passthru; stdenv.mkDerivation {
   pname = "python3";
   inherit version;
@@ -166,6 +204,13 @@ in with passthru; stdenv.mkDerivation {
   ] ++ [
     # LDSHARED now uses $CC instead of gcc. Fixes cross-compilation of extension modules.
     ./3.8/0001-On-all-posix-systems-not-just-Darwin-set-LDSHARED-if.patch
+    # Use sysconfigdata to find headers. Fixes cross-compilation of extension modules.
+    (
+      if isPy36 then
+        ./3.6/fix-finding-headers-when-cross-compiling.patch
+      else
+        ./3.7/fix-finding-headers-when-cross-compiling.patch
+    )
   ];
 
   postPatch = ''
@@ -279,6 +324,14 @@ in with passthru; stdenv.mkDerivation {
     find $out/lib/python*/config-* -type f -print -exec nuke-refs -e $out '{}' +
     find $out/lib -name '_sysconfigdata*.py*' -print -exec nuke-refs -e $out '{}' +
 
+    # Make the sysconfigdata module accessible on PYTHONPATH
+    # This allows build Python to import host Python's sysconfigdata
+    mkdir -p "$out/${sitePackages}"
+    mv "$out/lib/${libPrefix}"/_sysconfigdata*.py "$out/${sitePackages}"
+    if [ -d "$out/lib/${libPrefix}"/__pycache__ ]; then
+      mkdir -p "$out/${sitePackages}/__pycache__"
+      mv "$out/lib/${libPrefix}"/__pycache__/_sysconfigdata*.py* "$out/${sitePackages}/__pycache__"
+    fi
     '' + optionalString stripConfig ''
     rm -R $out/bin/python*-config $out/lib/python*/config-*
     '' + optionalString stripIdlelib ''
@@ -311,6 +364,14 @@ in with passthru; stdenv.mkDerivation {
     export PATH=${stdenv.lib.makeBinPath [ "$out" bash ]}:$PATH
   '';
 
+  # Add CPython specific setup-hook that configures distutils.sysconfig to
+  # always load sysconfigdata from host Python.
+  postFixup = ''
+    cat << "EOF" >> "$out/nix-support/setup-hook"
+    ${sysconfigdataHook}
+    EOF
+  '';
+
   # Enforce that we don't have references to the OpenSSL -dev package, which we
   # explicitly specify in our configure flags above.
   disallowedReferences =

pkgs/development/python-modules/bootstrapped-pip/default.nix

@@ -45,7 +45,7 @@ stdenv.mkDerivation rec {
     mv wheel* wheel
     # Set up PYTHONPATH. The above folders need to be on PYTHONPATH
     # $out is where we are installing to and takes precedence
-    export PYTHONPATH="$out/${python.sitePackages}:$(pwd)/pip/src:$(pwd)/setuptools:$(pwd)/setuptools/pkg_resources:$(pwd)/wheel"
+    export PYTHONPATH="$out/${python.sitePackages}:$(pwd)/pip/src:$(pwd)/setuptools:$(pwd)/setuptools/pkg_resources:$(pwd)/wheel:$PYTHONPATH"
 
     echo "Building setuptools wheel..."
     pushd setuptools