From 63cf61e8d37afd46992792a927f263a1ee92c673 Mon Sep 17 00:00:00 2001 From: Frederik Rietdijk Date: Thu, 4 Feb 2016 21:30:39 +0100 Subject: [PATCH 1/5] pythonPackages: new functions to build numpy and scipy --- .../python-modules/numpy-scipy-support.nix | 35 ------- pkgs/development/python-modules/numpy.nix | 51 ++++++++++ pkgs/development/python-modules/scipy.nix | 47 ++++++++++ pkgs/top-level/python-packages.nix | 94 ++++--------------- 4 files changed, 117 insertions(+), 110 deletions(-) delete mode 100644 pkgs/development/python-modules/numpy-scipy-support.nix create mode 100644 pkgs/development/python-modules/numpy.nix create mode 100644 pkgs/development/python-modules/scipy.nix diff --git a/pkgs/development/python-modules/numpy-scipy-support.nix b/pkgs/development/python-modules/numpy-scipy-support.nix deleted file mode 100644 index 422de794e31b..000000000000 --- a/pkgs/development/python-modules/numpy-scipy-support.nix +++ /dev/null @@ -1,35 +0,0 @@ -{ - # Python package expression - python, - # Name of package (e.g. numpy or scipy) - pkgName, - # OpenBLAS math library - openblas -}: - -{ - # Re-export openblas here so that it can be sure that the same one will be used - # in the propagatedBuildInputs. - inherit openblas; - - # First "install" the package, then import what was installed, and call the - # .test() function, which will run the test suite. - checkPhase = '' - runHook preCheck - pushd dist - ${python.interpreter} -c 'import ${pkgName}; ${pkgName}.test("fast", verbose=10)' - popd - runHook postCheck - ''; - - # Creates a site.cfg telling the setup script where to find depended-on - # math libraries. - preBuild = '' - echo "Creating site.cfg file..." - cat << EOF > site.cfg - [openblas] - include_dirs = ${openblas}/include - library_dirs = ${openblas}/lib - EOF - ''; -} diff --git a/pkgs/development/python-modules/numpy.nix b/pkgs/development/python-modules/numpy.nix new file mode 100644 index 000000000000..141c8b14fa6d --- /dev/null +++ b/pkgs/development/python-modules/numpy.nix @@ -0,0 +1,51 @@ +{lib, python, buildPythonPackage, isPyPy, gfortran, nose, blas}: + +args: + +let + inherit (args) version; +in buildPythonPackage (args // rec { + + name = "numpy-${version}"; + + disabled = isPyPy; + buildInputs = args.buildInputs or [ gfortran nose ]; + propagatedBuildInputs = args.propagatedBuildInputs or [ passthru.blas ]; + + preConfigure = '' + sed -i 's/-faltivec//' numpy/distutils/system_info.py + ''; + + preBuild = '' + echo "Creating site.cfg file..." + cat << EOF > site.cfg + [openblas] + include_dirs = ${passthru.blas}/include + library_dirs = ${passthru.blas}/lib + EOF + ''; + + checkPhase = '' + runHook preCheck + pushd dist + ${python.interpreter} -c 'import numpy; numpy.test("fast", verbose=10)' + popd + runHook postCheck + ''; + + passthru = { + blas = blas; + }; + + # The large file support test is disabled because it takes forever + # and can cause the machine to run out of disk space when run. + prePatch = '' + sed -i 's/test_large_file_support/donttest/' numpy/lib/tests/test_format.py + ''; + + meta = { + description = "Scientific tools for Python"; + homepage = "http://numpy.scipy.org/"; + maintainers = with lib.maintainers; [ fridh ]; + } // (args.meta or {}); +}) diff --git a/pkgs/development/python-modules/scipy.nix b/pkgs/development/python-modules/scipy.nix new file mode 100644 index 000000000000..ae312cd32d59 --- /dev/null +++ b/pkgs/development/python-modules/scipy.nix @@ -0,0 +1,47 @@ +{lib, python, buildPythonPackage, isPyPy, gfortran, nose}: + +args: + +let + inherit (args) version; + inherit (args) numpy; +in buildPythonPackage (args // rec { + + name = "scipy-${version}"; + + buildInputs = (args.buildInputs or [ gfortran nose ]); + propagatedBuildInputs = (args.propagatedBuildInputs or [ passthru.blas numpy]); + + preConfigure = '' + sed -i '0,/from numpy.distutils.core/s//import setuptools;from numpy.distutils.core/' setup.py + ''; + + preBuild = '' + echo "Creating site.cfg file..." + cat << EOF > site.cfg + [openblas] + include_dirs = ${passthru.blas}/include + library_dirs = ${passthru.blas}/lib + EOF + ''; + + checkPhase = '' + runHook preCheck + pushd dist + ${python.interpreter} -c 'import scipy; scipy.test("fast", verbose=10)' + popd + runHook postCheck + ''; + + passthru = { + blas = numpy.blas; + }; + + setupPyBuildFlags = [ "--fcompiler='gnu95'" ]; + + meta = { + description = "SciPy (pronounced 'Sigh Pie') is open-source software for mathematics, science, and engineering. "; + homepage = http://www.scipy.org/; + maintainers = with lib.maintainers; [ fridh ]; + } // (args.meta or {}); +}) diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 18671f3b4c73..65a8279ec92d 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -12505,48 +12505,20 @@ in modules // { }; }; - numpy = let - support = import ../development/python-modules/numpy-scipy-support.nix { - inherit python; - openblas = pkgs.openblasCompat; - pkgName = "numpy"; - }; - in buildPythonPackage ( rec { - name = "numpy-${version}"; - version = "1.10.4"; + buildNumpyPackage = callPackage ../development/python-modules/numpy.nix { + gfortran = pkgs.gfortran; + blas = pkgs.openblasCompat; + }; + numpy = self.numpy_1_10; + + numpy_1_10 = self.buildNumpyPackage rec { + version = "1.10.4"; src = pkgs.fetchurl { - url = "https://pypi.python.org/packages/source/n/numpy/${name}.tar.gz"; + url = "https://pypi.python.org/packages/source/n/numpy/numpy-${version}.tar.gz"; sha256 = "7356e98fbcc529e8d540666f5a919912752e569150e9a4f8d869c686f14c720b"; }; - - disabled = isPyPy; # WIP - - preConfigure = '' - sed -i 's/-faltivec//' numpy/distutils/system_info.py - ''; - - inherit (support) preBuild checkPhase; - - buildInputs = [ pkgs.gfortran self.nose ]; - propagatedBuildInputs = [ support.openblas ]; - - # Disable failing test_f2py test. - # f2py couldn't be found by test, - # even though it was used successfully to build numpy - - # The large file support test is disabled because it takes forever - # and can cause the machine to run out of disk space when run. - prePatch = '' - sed -i 's/test_f2py/donttest/' numpy/tests/test_scripts.py - sed -i 's/test_large_file_support/donttest/' numpy/lib/tests/test_format.py - ''; - - meta = { - description = "Scientific tools for Python"; - homepage = "http://numpy.scipy.org/"; - }; - }); + }; numpydoc = buildPythonPackage rec { name = "numpydoc-${version}"; @@ -18488,47 +18460,19 @@ in modules // { }; }; + buildScipyPackage = callPackage ../development/python-modules/scipy.nix { + gfortran = pkgs.gfortran; + }; - scipy = let - support = import ../development/python-modules/numpy-scipy-support.nix { - inherit python; - openblas = pkgs.openblasCompat; - pkgName = "scipy"; - }; - in buildPythonPackage rec { - name = "scipy-${version}"; + scipy = self.scipy_0_16; + + scipy_0_16 = self.buildScipyPackage rec { version = "0.16.1"; - src = pkgs.fetchurl { - url = "http://pypi.python.org/packages/source/s/scipy/${name}.tar.gz"; + url = "https://pypi.python.org/packages/source/s/scipy/scipy-${version}.tar.gz"; sha256 = "ecd1efbb1c038accb0516151d1e6679809c6010288765eb5da6051550bf52260"; }; - - buildInputs = [ pkgs.gfortran self.nose ]; - propagatedBuildInputs = [ self.numpy ]; - - preConfigure = '' - sed -i '0,/from numpy.distutils.core/s//import setuptools;from numpy.distutils.core/' setup.py - ''; - - # First test: RuntimeWarning: Mean of empty slice. - # Second: SyntaxError: invalid syntax. Due to wrapper? - # Third: test checks permissions - prePatch = '' - substituteInPlace scipy/stats/tests/test_stats.py --replace "test_chisquare_masked_arrays" "remove_this_one" - rm scipy/linalg/tests/test_lapack.py - substituteInPlace scipy/weave/tests/test_catalog.py --replace "test_user" "remove_this_one" - ''; - - inherit (support) preBuild checkPhase; - - patches = [../development/python-modules/scipy-0.16.1-decorator-fix.patch]; - setupPyBuildFlags = [ "--fcompiler='gnu95'" ]; - - meta = { - description = "SciPy (pronounced 'Sigh Pie') is open-source software for mathematics, science, and engineering. "; - homepage = http://www.scipy.org/; - }; + numpy = self.numpy_1_10; }; scikitimage = buildPythonPackage rec { @@ -18563,7 +18507,7 @@ in modules // { }; buildInputs = with self; [ nose pillow pkgs.gfortran pkgs.glibcLocales ]; - propagatedBuildInputs = with self; [ numpy scipy pkgs.openblas ]; + propagatedBuildInputs = with self; [ numpy scipy numpy.blas ]; LC_ALL="en_US.UTF-8"; From 3e476a73ef455484e7b60a36364bd29fd60bfd4a Mon Sep 17 00:00:00 2001 From: Frederik Rietdijk Date: Thu, 4 Feb 2016 21:29:25 +0100 Subject: [PATCH 2/5] openblas_2_14: init at 0.2.14 --- .../science/math/openblas/0.2.14.nix | 65 +++++++++++++++++++ pkgs/top-level/all-packages.nix | 2 + 2 files changed, 67 insertions(+) create mode 100644 pkgs/development/libraries/science/math/openblas/0.2.14.nix diff --git a/pkgs/development/libraries/science/math/openblas/0.2.14.nix b/pkgs/development/libraries/science/math/openblas/0.2.14.nix new file mode 100644 index 000000000000..2fac8a4db08b --- /dev/null +++ b/pkgs/development/libraries/science/math/openblas/0.2.14.nix @@ -0,0 +1,65 @@ +{ stdenv, fetchurl, gfortran, perl, which, config, coreutils +# Most packages depending on openblas expect integer width to match pointer width, +# but some expect to use 32-bit integers always (for compatibility with reference BLAS). +, blas64 ? null +}: + +with stdenv.lib; + +let blas64_ = blas64; in + +let local = config.openblas.preferLocalBuild or false; + binary = + { i686-linux = "32"; + x86_64-linux = "64"; + x86_64-darwin = "64"; + }."${stdenv.system}" or (throw "unsupported system: ${stdenv.system}"); + genericFlags = + [ "DYNAMIC_ARCH=1" + "NUM_THREADS=64" + ]; + localFlags = config.openblas.flags or + optionals (hasAttr "target" config.openblas) [ "TARGET=${config.openblas.target}" ]; + blas64 = if blas64_ != null then blas64_ else hasPrefix "x86_64" stdenv.system; + + version = "0.2.14"; +in +stdenv.mkDerivation { + name = "openblas-${version}"; + src = fetchurl { + url = "https://github.com/xianyi/OpenBLAS/archive/v${version}.tar.gz"; + sha256 = "2411c4f56f477b42dff54db2b7ffc0b7cf53bb9778d54982595c64cc69c40fc1"; + name = "openblas-${version}.tar.gz"; + }; + + inherit blas64; + + nativeBuildInputs = optionals stdenv.isDarwin [coreutils] ++ [gfortran perl which]; + + makeFlags = + (if local then localFlags else genericFlags) + ++ + optionals stdenv.isDarwin ["MACOSX_DEPLOYMENT_TARGET=10.9"] + ++ + [ + "FC=gfortran" + # Note that clang is available through the stdenv on OSX and + # thus is not an explicit dependency. + "CC=${if stdenv.isDarwin then "clang" else "gcc"}" + ''PREFIX="''$(out)"'' + "BINARY=${binary}" + "USE_OPENMP=${if stdenv.isDarwin then "0" else "1"}" + "INTERFACE64=${if blas64 then "1" else "0"}" + ]; + + doCheck = true; + checkTarget = "tests"; + + meta = with stdenv.lib; { + description = "Basic Linear Algebra Subprograms"; + license = licenses.bsd3; + homepage = "https://github.com/xianyi/OpenBLAS"; + platforms = platforms.unix; + maintainers = with maintainers; [ ttuegel ]; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index e103d1e5b536..2eec8c8d3dba 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -15029,10 +15029,12 @@ let liblbfgs = callPackage ../development/libraries/science/math/liblbfgs { }; openblas = callPackage ../development/libraries/science/math/openblas { }; + openblas_2_14 = callPackage ../development/libraries/science/math/openblas/0.2.14.nix { }; # A version of OpenBLAS using 32-bit integers on all platforms for compatibility with # standard BLAS and LAPACK. openblasCompat = openblas.override { blas64 = false; }; + openblasCompat_2_14 = openblas_2_14.override { blas64 = false; }; openlibm = callPackage ../development/libraries/science/math/openlibm {}; From 73e9cb9748bdd5e3343972aafd094669d629ff3a Mon Sep 17 00:00:00 2001 From: Frederik Rietdijk Date: Fri, 5 Feb 2016 09:03:11 +0100 Subject: [PATCH 3/5] pythonPackages.numpy: use openblas_2_14 --- pkgs/top-level/python-packages.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 65a8279ec92d..16fc9ad808c8 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -12507,7 +12507,7 @@ in modules // { buildNumpyPackage = callPackage ../development/python-modules/numpy.nix { gfortran = pkgs.gfortran; - blas = pkgs.openblasCompat; + blas = pkgs.openblasCompat_2_14; }; numpy = self.numpy_1_10; From 7fad1bcff7af266d020692f804ae7d3b80a81697 Mon Sep 17 00:00:00 2001 From: Frederik Rietdijk Date: Mon, 8 Feb 2016 20:36:01 +0100 Subject: [PATCH 4/5] pythonPackages.scipy_0_17: init at 0.17.0 --- pkgs/top-level/python-packages.nix | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 16fc9ad808c8..f4d6bd965a0f 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -18475,6 +18475,15 @@ in modules // { numpy = self.numpy_1_10; }; + scipy_0_17 = self.buildScipyPackage rec { + version = "0.17.0"; + src = pkgs.fetchurl { + url = "https://pypi.python.org/packages/source/s/scipy/scipy-${version}.tar.gz"; + sha256 = "f600b755fb69437d0f70361f9e560ab4d304b1b66987ed5a28bdd9dd7793e089"; + }; + numpy = self.numpy_1_10; + }; + scikitimage = buildPythonPackage rec { name = "scikit-image-${version}"; version = "0.11.3"; From 9123a84fd32071257931a97f708c56603840cc80 Mon Sep 17 00:00:00 2001 From: Frederik Rietdijk Date: Fri, 5 Feb 2016 09:00:50 +0100 Subject: [PATCH 5/5] pythonPackages.scipy: 0.16.1 -> 0.17.0 --- pkgs/top-level/python-packages.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index f4d6bd965a0f..55e36cbe71e9 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -18464,7 +18464,7 @@ in modules // { gfortran = pkgs.gfortran; }; - scipy = self.scipy_0_16; + scipy = self.scipy_0_17; scipy_0_16 = self.buildScipyPackage rec { version = "0.16.1";