alioth/nixpkgs
3
0
Fork 0
forked from mirrors/nixpkgs
Code Releases Activity

Merge pull request #75827 from stites/pytorch-1.3.1

Browse source 
python3Packages.pytorch: 1.2.0 -> 1.4.1, python3Packages.ignite: 0.2.1 -> 0.3.0
This commit is contained in:
Benjamin Hipple 2020-05-09 15:27:27 -04:00 committed by GitHub
parent b907387ffe ecef2975e3
commit 3d9f3c3dd3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 88 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
pkgs/development/python-modules/ignite/default.nix
View file

@ -2,6 +2,7 @@
, buildPythonPackage
, fetchFromGitHub
, pytest
, matplotlib
, mock
, pytorch
, pynvml
@ -11,24 +12,24 @@
buildPythonPackage rec {
pname = "ignite";
version = "0.2.1";
version = "0.3.0";
src = fetchFromGitHub {
owner = "pytorch";
repo = pname;
rev = "v${version}";
sha256 = "15k6dd11yxn4923llcpmw4srl1by5ljhh7aw5pnkn4n4qpywh6cm";
sha256 = "0i863kxi1r1hspj19lhn6r8256vdazjcyvis0s33fgzrf7kxi08x";
};
checkInputs = [ pytest mock ];
checkPhase = ''
pytest -k 'not visdom and not tensorboard and not mlflow and not polyaxon' tests/
'';
# these packages are not currently in nixpkgs
checkInputs = [ pytest matplotlib mock ];
propagatedBuildInputs = [ pytorch scikitlearn tqdm pynvml ];
# Some packages are not in NixPkgs; other tests try to build distributed
# models, which doesn't work in the sandbox.
checkPhase = ''
pytest -k 'not visdom and not tensorboard and not mlflow and not polyaxon and not conftest and not engines and not distrib_' tests/
'';
meta = with lib; {
description = "High-level training library for PyTorch";
homepage = "https://pytorch.org/ignite";

112
pkgs/development/python-modules/pytorch/default.nix
View file

@ -1,23 +1,25 @@
{ stdenv, fetchurl, fetchgit, buildPythonPackage, python, pythonOlder,
{ stdenv, lib, fetchFromGitHub, fetchpatch, buildPythonPackage, python,
cudaSupport ? false, cudatoolkit ? null, cudnn ? null, nccl ? null, magma ? null,
mklSupport ? false, mkl ? null,
mklDnnSupport ? true, useSystemNccl ? true,
openMPISupport ? false, openmpi ? null,
buildNamedTensor ? false,
buildBinaries ? false,
buildDocs ? false,
cudaArchList ? null,
fetchFromGitHub, lib, numpy, pyyaml, cffi, click, typing, cmake, hypothesis, numactl,
numpy, pyyaml, cffi, click, typing, cmake, oneDNN, hypothesis, numactl, psutil,
linkFarm, symlinkJoin,
# virtual pkg that consistently instantiates blas across nixpkgs
# See https://github.com/NixOS/nixpkgs/pull/83888
blas,
# ninja (https://ninja-build.org) must be available to run C++ extensions tests,
ninja,
# dependencies for torch.utils.tensorboard
tensorboardSupport ? true, pillow, six, future, tensorflow-tensorboard,
pillow, six, future, tensorflow-tensorboard, protobuf,
utillinux, which, isPy3k }:
assert !openMPISupport || openmpi != null;
assert !tensorboardSupport || tensorflow-tensorboard != null;
# assert that everything needed for cuda is present and that the correct cuda versions are used
assert !cudaSupport || cudatoolkit != null;
@ -28,17 +30,11 @@ assert !cudaSupport || (let majorIs = lib.versions.major cudatoolkit.version;
let
hasDependency = dep: pkg: lib.lists.any (inp: inp == dep) pkg.buildInputs;
matchesCudatoolkit = hasDependency cudatoolkit;
matchesMkl = hasDependency mkl;
in
# confirm that cudatoolkits are sync'd across dependencies
assert !(openMPISupport && cudaSupport) || matchesCudatoolkit openmpi;
assert !cudaSupport || matchesCudatoolkit magma;
# confirm that mkl is sync'd across dependencies
assert !mklSupport || mkl != null;
assert !(mklSupport && cudaSupport) || matchesMkl magma;
assert !mklSupport || (numpy.blasImplementation == "mkl" && numpy.blas == mkl);
let
cudatoolkit_joined = symlinkJoin {
name = "${cudatoolkit.name}-unsplit";
@ -108,7 +104,7 @@ let
"LD_LIBRARY_PATH=${cudaStub}\${LD_LIBRARY_PATH:+:}$LD_LIBRARY_PATH ";
in buildPythonPackage rec {
version = "1.2.0";
version = "1.4.1";
pname = "pytorch";
disabled = !isPy3k;
@ -122,11 +118,9 @@ in buildPythonPackage rec {
repo = "pytorch";
rev = "v${version}";
fetchSubmodules = true;
sha256 = "1biyq2p48chakf2xw7hazzqmr5ps1nx475ql8vkmxjg5zaa071cz";
sha256 = "1aa1il4f98pswfj20cv27yfb91l1jcq4515i7mvq7sh5647yzwms";
};
dontUseCmakeConfigure = true;
preConfigure = lib.optionalString cudaSupport ''
export TORCH_CUDA_ARCH_LIST="${lib.strings.concatStringsSep ";" final_cudaArchList}"
export CC=${cudatoolkit.cc}/bin/gcc CXX=${cudatoolkit.cc}/bin/g++
@ -134,6 +128,44 @@ in buildPythonPackage rec {
export CUDNN_INCLUDE_DIR=${cudnn}/include
'';
patches = [
# Prevents a race condition which would be introduced by pull 30333.
# See https://github.com/pytorch/pytorch/issues/32277
# Can be removed >1.5.0.
(fetchpatch {
url = "https://patch-diff.githubusercontent.com/raw/pytorch/pytorch/pull/30332.patch";
sha256 = "1v9dwbhz3rdxcx6sz8y8j9n3bj6nqs78b1r8yg89yc15n6l4cqx2";
})
# Fixes errors with gcc-9 compilation. Cherry-picked on advice from ezyang.
# See https://github.com/pytorch/pytorch/issues/32277
# Can be removed >1.5.0.
(fetchpatch {
url = "https://patch-diff.githubusercontent.com/raw/pytorch/pytorch/pull/30333.patch";
sha256 = "139413fl37h2fnil0cv99a67mqqnsh02k74b92by1qyr6pcfyg3q";
})
];
# Use pytorch's custom configurations
dontUseCmakeConfigure = true;
BUILD_NAMEDTENSOR = true;
BUILD_DOCS = buildDocs;
USE_MKL = blas.implementation == "mkl";
# Unlike MKL, MKLDNN is FOSS, so we enable support for it by default. Note
# that this was renamed to dnnl and then renamed again to oneDNN upstream, but
# pytorch still calls it by the old name mkldnn.
USE_MKLDNN = mklDnnSupport;
USE_MKLDNN_CBLAS = mklDnnSupport;
preBuild = ''
export MAX_JOBS=$NIX_BUILD_CORES
${python.interpreter} setup.py build --cmake-only
${cmake}/bin/cmake build
'';
preFixup = ''
function join_by { local IFS="$1"; shift; echo "$*"; }
function strip2 {
@ -155,8 +187,7 @@ in buildPythonPackage rec {
PYTORCH_BUILD_VERSION = version;
PYTORCH_BUILD_NUMBER = 0;
BUILD_NAMEDTENSOR = buildNamedTensor; # experimental feature
USE_SYSTEM_NCCL=true; # don't build pytorch's third_party NCCL
USE_SYSTEM_NCCL=useSystemNccl; # don't build pytorch's third_party NCCL
# Suppress a weird warning in mkl-dnn, part of ideep in pytorch
# (upstream seems to have fixed this in the wrong place?)
@ -165,7 +196,7 @@ in buildPythonPackage rec {
#
# Also of interest: pytorch ignores CXXFLAGS uses CFLAGS for both C and C++:
# https://github.com/pytorch/pytorch/blob/v1.2.0/setup.py#L17
NIX_CFLAGS_COMPILE = lib.optionals (numpy.blas == mkl) [ "-Wno-error=array-bounds" ];
NIX_CFLAGS_COMPILE = lib.optionals (blas.implementation == "mkl") [ "-Wno-error=array-bounds" ];
nativeBuildInputs = [
cmake
@ -174,9 +205,8 @@ in buildPythonPackage rec {
ninja
] ++ lib.optionals cudaSupport [ cudatoolkit_joined ];
buildInputs = [
numpy.blas
] ++ lib.optionals cudaSupport [ cudnn magma nccl ]
buildInputs = [ blas blas.provider oneDNN ]
++ lib.optionals cudaSupport [ cudnn magma nccl ]
++ lib.optionals stdenv.isLinux [ numactl ];
propagatedBuildInputs = [
@ -184,23 +214,37 @@ in buildPythonPackage rec {
click
numpy
pyyaml
] ++ lib.optionals openMPISupport [ openmpi ]
++ lib.optional (pythonOlder "3.5") typing
++ lib.optionals tensorboardSupport [pillow six future tensorflow-tensorboard];
# the following are required for tensorboard support
pillow six future tensorflow-tensorboard protobuf
] ++ lib.optionals openMPISupport [ openmpi ];
checkInputs = [ hypothesis ninja ];
checkInputs = [ hypothesis ninja psutil ];
doCheck = false; # tests take a long time for channel release, so doCheck should be overridden only when developing
checkPhase = "${cudaStubEnv}python test/run_test.py"
+ " --exclude utils" # utils requires git, which is not allowed in the check phase
# Tests take a long time and may be flaky, so just sanity-check imports
doCheck = false;
pythonImportsCheck = [
"torch"
];
# Other tests which have been disabled in previous nix derivations of pytorch.
# --exclude dataloader sparse torch utils thd_distributed distributed cpp_extensions
;
checkPhase = with lib.versions; with lib.strings; concatStringsSep " " [
cudaStubEnv
"${python.interpreter} test/run_test.py"
"--exclude"
(concatStringsSep " " [
"utils" # utils requires git, which is not allowed in the check phase
# "dataloader" # psutils correctly finds and triggers multiprocessing, but is too sandboxed to run -- resulting in numerous errors
# ^^^^^^^^^^^^ NOTE: while test_dataloader does return errors, these are acceptable errors and do not interfere with the build
# tensorboard has acceptable failures for pytorch 1.3.x due to dependencies on tensorboard-plugins
(optionalString (majorMinor version == "1.3" ) "tensorboard")
])
];
postInstall = ''
mkdir $dev
cp -r $out/${python.sitePackages}/torch/lib $dev/lib
cp -r $out/${python.sitePackages}/torch/include $dev/include
cp -r $out/${python.sitePackages}/torch/share $dev/share
'';
postFixup = stdenv.lib.optionalString stdenv.isDarwin ''
@ -233,6 +277,6 @@ in buildPythonPackage rec {
homepage = "https://pytorch.org/";
license = lib.licenses.bsd3;
platforms = with lib.platforms; linux ++ lib.optionals (!cudaSupport) darwin;
maintainers = with lib.maintainers; [ teh thoughtpolice stites tscholak ]; # tscholak esp. for darwin-related builds
maintainers = with lib.maintainers; [ teh thoughtpolice tscholak ]; # tscholak esp. for darwin-related builds
};
}