mkl: include Intel's libiomp.so in the MKL RPM unpack

Since Intel's default openmp implementation is available in the same src
tarball, we can just include it in the package. This means that `mkl` now "just
works" without any environment variables, fragile setup-hooks, or forced
propagation.

Since the openmp implementation is only needed at runtime (and for test cases),
users can substitute a different one if they prefer by exporting it with
`LD_PRELOAD`, which is how Intel recommends handling this. If they do not do so,
`libiomp.so` lives next to `libmkl_rt.so` and thus will be in the RPATH as a
sane default.

Since this still comes from the same src tarball, we can ship it without losing
the fixed-output derivation; likewise, since Hydra is not building or caching
these, shipping these proprietary packages costs no bandwidth for the nix
community.

doc/languages-frameworks/python.section.md

@@ -1104,7 +1104,7 @@ on `numpy` will be built with `mkl`.
 The following is an overlay that configures `numpy` to use `mkl`:
 ```nix
 self: super: {
-  python36 = super.python36.override {
+  python37 = super.python37.override {
     packageOverrides = python-self: python-super: {
       numpy = python-super.numpy.override {
         blas = super.pkgs.mkl;
@@ -1114,6 +1114,15 @@ self: super: {
 }
 ```
 
+`mkl` requires an `openmp` implementation when running with multiple processors.
+By default, `mkl` will use Intel's `iomp` implementation if no other is
+specified, but this is a runtime-only dependency and binary compatible with the
+LLVM implementation. To use that one instead, Intel recommends users set it with
+`LD_PRELOAD`.
+
+Note that `mkl` is only available on `x86_64-{linux,darwin}` platforms;
+moreover, Hydra is not building and distributing pre-compiled binaries using it.
+
 ## Contributing
 
 ### Contributing guidelines

pkgs/development/libraries/science/math/mkl/default.nix

@@ -1,21 +1,8 @@
 { stdenvNoCC, writeText, fetchurl, rpmextract, undmg }:
 /*
-  Some (but not all) mkl functions require openmp, but Intel does not add these
-  to SO_NEEDED and instructs users to put openmp on their LD_LIBRARY_PATH. If
-  you are using mkl and your library/application is using some of the functions
-  that require openmp, add a setupHook like this to your package:
-
-  setupHook = writeText "setup-hook.sh" ''
-    addOpenmp() {
-        addToSearchPath LD_LIBRARY_PATH ${openmp}/lib
-    }
-    addEnvHooks "$targetOffset" addOpenmp
-  '';
-
-  We do not add the setup hook here, because avoiding it allows this large
-  package to be a fixed-output derivation with better cache efficiency.
- */
-
+  For details on using mkl as a blas provider for python packages such as numpy,
+  numexpr, scipy, etc., see the Python section of the NixPkgs manual.
+*/
 stdenvNoCC.mkDerivation rec {
   name = "mkl-${version}";
   version = "${date}.${rel}";
@@ -43,16 +30,23 @@ stdenvNoCC.mkDerivation rec {
   '' else ''
     rpmextract rpm/intel-mkl-common-c-${date}-${rel}-${date}-${rel}.noarch.rpm
     rpmextract rpm/intel-mkl-core-rt-${date}-${rel}-${date}-${rel}.x86_64.rpm
+    rpmextract rpm/intel-openmp-19.0.0-${rel}-19.0.0-${rel}.x86_64.rpm
   '';
 
   installPhase = if stdenvNoCC.isDarwin then ''
       mkdir -p $out/lib
+
       cp -r compilers_and_libraries_${version}/mac/mkl/include $out/
-      cp -r compilers_and_libraries_${version}/mac/mkl/lib/* $out/lib/
+
       cp -r compilers_and_libraries_${version}/licensing/mkl/en/license.txt $out/lib/
+      cp -r compilers_and_libraries_${version}/mac/compiler/lib/* $out/lib/
+      cp -r compilers_and_libraries_${version}/mac/mkl/lib/* $out/lib/
   '' else ''
       mkdir -p $out/lib
+
       cp -r opt/intel/compilers_and_libraries_${version}/linux/mkl/include $out/
+
+      cp -r opt/intel/compilers_and_libraries_${version}/linux/compiler/lib/intel64_lin/* $out/lib/
       cp -r opt/intel/compilers_and_libraries_${version}/linux/mkl/lib/intel64_lin/* $out/lib/
       cp license.txt $out/lib/
   '';
@@ -66,8 +60,8 @@ stdenvNoCC.mkDerivation rec {
   outputHashAlgo = "sha256";
   outputHashMode = "recursive";
   outputHash = if stdenvNoCC.isDarwin
-    then "1224dln7n8px1rk8biiggf77wjhxh8mzw0hd8zlyjm8i6j8w7i12"
-    else "0d8ai0wi8drp071acqkm1wv6vyg12010y843y56zzi1pql81xqvx";
+    then "0000000000000000000000000000000000000000000000000000"
+    else "1amagcaan0hk3x9v7gg03gkw02n066v4kmjb32yyzsy5rfrivb1a";
 
   meta = with stdenvNoCC.lib; {
     description = "Intel Math Kernel Library";
@@ -78,7 +72,7 @@ stdenvNoCC.mkDerivation rec {
       threading models.
     '';
     homepage = https://software.intel.com/en-us/mkl;
-    license = [ licenses.issl licenses.unfreeRedistributable ];
+    license = licenses.issl;
     platforms = [ "x86_64-linux" "x86_64-darwin" ];
     maintainers = [ maintainers.bhipple ];
   };

pkgs/development/python-modules/numexpr/default.nix

@@ -3,7 +3,6 @@
 , fetchPypi
 , python
 , numpy
-, llvmPackages ? null
 }:
 
 buildPythonPackage rec {
@@ -16,16 +15,11 @@ buildPythonPackage rec {
   };
 
   # Remove existing site.cfg, use the one we built for numpy.
-  # Somehow openmp needs to be added to LD_LIBRARY_PATH
-  # https://software.intel.com/en-us/forums/intel-system-studio/topic/611682
   preBuild = ''
     rm site.cfg
     ln -s ${numpy.cfg} site.cfg
-    export LD_LIBRARY_PATH=${llvmPackages.openmp}/lib
   '';
 
-  buildInputs = [] ++ lib.optional (numpy.blasImplementation == "mkl") llvmPackages.openmp;
-
   propagatedBuildInputs = [ numpy ];
 
   # Run the test suite.
@@ -47,4 +41,4 @@ buildPythonPackage rec {
     homepage = "https://github.com/pydata/numexpr";
     license = lib.licenses.mit;
   };
-}
+}

pkgs/development/python-modules/numpy/default.nix

@@ -71,8 +71,6 @@ in buildPythonPackage rec {
     inherit blasImplementation cfg;
   };
 
-  doCheck = blasImplementation != "mkl";
-
   # Disable two tests
   # - test_f2py: f2py isn't yet on path.
   # - test_large_file_support: takes a long time and can cause the machine to run out of disk space