Merge pull request #51733 from averelld/nvidia-docker

nvidia-docker module/package

nixos/modules/hardware/video/nvidia.nix

@@ -172,6 +172,11 @@ in
     environment.systemPackages = [ nvidia_x11.bin nvidia_x11.settings ]
       ++ lib.filter (p: p != null) [ nvidia_x11.persistenced ];
 
+    systemd.tmpfiles.rules = optional config.virtualisation.docker.enableNvidia
+        "L+ /run/nvidia-docker/bin - - - - ${nvidia_x11.bin}/origBin"
+      ++ optional (nvidia_x11.persistenced != null && config.virtualisation.docker.enableNvidia)
+        "L+ /run/nvidia-docker/extras/bin/nvidia-persistenced - - - - ${nvidia_x11.persistenced}/origBin/nvidia-persistenced";
+
     boot.extraModulePackages = [ nvidia_x11.bin ];
 
     # nvidia-uvm is required by CUDA applications.

nixos/modules/virtualisation/docker.nix

@@ -52,6 +52,15 @@ in
           '';
       };
 
+    enableNvidia =
+      mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Enable nvidia-docker wrapper, supporting NVIDIA GPUs inside docker containers.
+        '';
+      };
+
     liveRestore =
       mkOption {
         type = types.bool;
@@ -140,7 +149,8 @@ in
   ###### implementation
 
   config = mkIf cfg.enable (mkMerge [{
-      environment.systemPackages = [ cfg.package ];
+      environment.systemPackages = [ cfg.package ]
+        ++ optional cfg.enableNvidia pkgs.nvidia-docker;
       users.groups.docker.gid = config.ids.gids.docker;
       systemd.packages = [ cfg.package ];
 
@@ -157,6 +167,7 @@ in
                 --log-driver=${cfg.logDriver} \
                 ${optionalString (cfg.storageDriver != null) "--storage-driver=${cfg.storageDriver}"} \
                 ${optionalString cfg.liveRestore "--live-restore" } \
+                ${optionalString cfg.enableNvidia "--add-runtime nvidia=${pkgs.nvidia-docker}/bin/nvidia-container-runtime" } \
                 ${cfg.extraOptions}
             ''];
           ExecReload=[
@@ -165,7 +176,8 @@ in
           ];
         };
 
-        path = [ pkgs.kmod ] ++ (optional (cfg.storageDriver == "zfs") pkgs.zfs);
+        path = [ pkgs.kmod ] ++ optional (cfg.storageDriver == "zfs") pkgs.zfs
+          ++ optional cfg.enableNvidia pkgs.nvidia-docker;
       };
 
       systemd.sockets.docker = {
@@ -179,7 +191,6 @@ in
         };
       };
 
-
       systemd.services.docker-prune = {
         description = "Prune docker resources";
 
@@ -194,7 +205,15 @@ in
 
         startAt = optional cfg.autoPrune.enable cfg.autoPrune.dates;
       };
+
+      assertions = [
+        { assertion = cfg.enableNvidia -> config.hardware.opengl.driSupport32Bit or false;
+          message = "Option enableNvidia requires 32bit support libraries";
+        }];
     }
+    (mkIf cfg.enableNvidia {
+      environment.etc."nvidia-container-runtime/config.toml".source = "${pkgs.nvidia-docker}/etc/config.toml";
+    })
   ]);
 
   imports = [

pkgs/applications/virtualization/nvidia-docker/config.toml

@@ -0,0 +1,13 @@
+disable-require = false
+#swarm-resource = "DOCKER_RESOURCE_GPU"
+
+[nvidia-container-cli]
+#root = "/run/nvidia/driver"
+#path = "/usr/bin/nvidia-container-cli"
+environment = []
+#debug = "/var/log/nvidia-container-runtime-hook.log"
+ldcache = "/tmp/ld.so.cache"
+load-kmods = true
+#no-cgroups = false
+#user = "root:video"
+ldconfig = "@@glibcbin@/bin/ldconfig"

pkgs/applications/virtualization/nvidia-docker/default.nix

@@ -0,0 +1,84 @@
+{ stdenv, lib, fetchFromGitHub, fetchpatch, callPackage, makeWrapper
+, buildGoPackage, runc, libelf, libcap, libseccomp, glibc }:
+
+with lib; let
+
+  glibc-ldconf = glibc.overrideAttrs (oldAttrs: {
+    # ldconfig needs help reading libraries that have been patchelf-ed, as the
+    # .dynstr section is no longer in the first LOAD segment. See also
+    # https://sourceware.org/bugzilla/show_bug.cgi?id=23964 and
+    # https://github.com/NixOS/patchelf/issues/44
+    patches = oldAttrs.patches ++ [ (fetchpatch {
+      name = "ldconfig-patchelf.patch";
+      url = "https://sourceware.org/bugzilla/attachment.cgi?id=11444";
+      sha256 = "0nzzmq7pli37iyjrgcmvcy92piiwjybpw245ds7q43pbgdm7lc3s";
+    })];
+  });
+
+  libnvidia-container = callPackage ./libnvc.nix { };
+
+  nvidia-container-runtime = fetchFromGitHub {
+    owner = "NVIDIA";
+    repo = "nvidia-container-runtime";
+    rev = "runtime-v2.0.0";
+    sha256 = "0jcj5xxbg7x7gyhbb67h3ds6vly62gx7j02zm6lg102h34jajj7a";
+  };
+
+  nvidia-container-runtime-hook = buildGoPackage rec {
+    name = "nvidia-container-runtime-hook-${version}";
+    version = "1.4.0";
+
+    goPackagePath = "nvidia-container-runtime-hook";
+
+    src = "${nvidia-container-runtime}/hook/nvidia-container-runtime-hook";
+  };
+
+  nvidia-runc = runc.overrideAttrs (oldAttrs: rec {
+    name = "nvidia-runc";
+    version = "1.0.0-rc6";
+    src = fetchFromGitHub {
+      owner = "opencontainers";
+      repo = "runc";
+      rev = "v${version}";
+      sha256 = "1jwacb8xnmx5fr86gximhbl9dlbdwj3rpf27hav9q1si86w5pb1j";
+    };
+    patches = [ "${nvidia-container-runtime}/runtime/runc/3f2f8b84a77f73d38244dd690525642a72156c64/0001-Add-prestart-hook-nvidia-container-runtime-hook-to-t.patch" ];
+  });
+
+in stdenv.mkDerivation rec {
+  name = "nvidia-docker-${version}";
+  version = "2.0.3";
+
+  src = fetchFromGitHub {
+    owner = "NVIDIA";
+    repo = "nvidia-docker";
+    rev = "v${version}";
+    sha256 = "1vx5m591mnvcb9vy0196x5lh3r8swjsk0fnlv5h62m7m4m07v6wx";
+  };
+
+  nativeBuildInputs = [ makeWrapper ];
+
+  buildPhase = ''
+    mkdir bin
+    cp nvidia-docker bin
+    cp ${libnvidia-container}/bin/nvidia-container-cli bin
+    cp ${nvidia-container-runtime-hook}/bin/nvidia-container-runtime-hook bin
+    cp ${nvidia-runc}/bin/runc bin/nvidia-container-runtime
+  '';
+
+  installPhase = ''
+    mkdir -p $out/{bin,etc}
+    cp -r bin $out
+    wrapProgram $out/bin/nvidia-container-cli \
+      --prefix LD_LIBRARY_PATH : /run/opengl-driver/lib:/run/opengl-driver-32/lib
+    cp ${./config.toml} $out/etc/config.toml
+    substituteInPlace $out/etc/config.toml --subst-var-by glibcbin ${lib.getBin glibc-ldconf}
+  '';
+
+  meta = {
+    homepage = https://github.com/NVIDIA/nvidia-docker;
+    description = "NVIDIA container runtime for Docker";
+    license = licenses.bsd3;
+    platforms = platforms.linux;
+  };
+}

pkgs/applications/virtualization/nvidia-docker/libnvc-ldconfig-and-path-fixes.patch

@@ -0,0 +1,130 @@
+diff --git a/src/ldcache.c b/src/ldcache.c
+index 38bab05..e1abc89 100644
+--- a/src/ldcache.c
++++ b/src/ldcache.c
+@@ -108,40 +108,27 @@ ldcache_close(struct ldcache *ctx)
+ 
+ int
+ ldcache_resolve(struct ldcache *ctx, uint32_t arch, const char *root, const char * const libs[],
+-    char *paths[], size_t size, ldcache_select_fn select, void *select_ctx)
++    char *paths[], size_t size, const char* version)
+ {
+         char path[PATH_MAX];
+-        struct header_libc6 *h;
+-        int override;
++        char dir[PATH_MAX];
++        char lib[PATH_MAX];
+ 
+-        h = (struct header_libc6 *)ctx->ptr;
+         memset(paths, 0, size * sizeof(*paths));
+ 
+-        for (uint32_t i = 0; i < h->nlibs; ++i) {
+-                int32_t flags = h->libs[i].flags;
+-                char *key = (char *)ctx->ptr + h->libs[i].key;
+-                char *value = (char *)ctx->ptr + h->libs[i].value;
+-
+-                if (!(flags & LD_ELF) || (flags & LD_ARCH_MASK) != arch)
++        for (size_t j = 0; j < size; ++j) {
++                snprintf(dir, 100, "/run/opengl-driver%s/lib",
++                    arch == LD_I386_LIB32 ? "-32" : "");
++                if (!strncmp(libs[j], "libvdpau_nvidia.so", 100))
++                  strcat(dir, "/vdpau");
++                snprintf(lib, 100, "%s/%s.%s", dir, libs[j], version);
++                if (path_resolve_full(ctx->err, path, "/", lib) < 0)
++                        return (-1);
++                if (!file_exists(ctx->err, path))
+                         continue;
+-
+-                for (size_t j = 0; j < size; ++j) {
+-                        if (!str_has_prefix(key, libs[j]))
+-                                continue;
+-                        if (path_resolve(ctx->err, path, root, value) < 0)
+-                                return (-1);
+-                        if (paths[j] != NULL && str_equal(paths[j], path))
+-                                continue;
+-                        if ((override = select(ctx->err, select_ctx, root, paths[j], path)) < 0)
+-                                return (-1);
+-                        if (override) {
+-                                free(paths[j]);
+-                                paths[j] = xstrdup(ctx->err, path);
+-                                if (paths[j] == NULL)
+-                                        return (-1);
+-                        }
+-                        break;
+-                }
++                paths[j] = xstrdup(ctx->err, path);
++                if (paths[j] == NULL)
++                        return (-1);
+         }
+         return (0);
+ }
+diff --git a/src/ldcache.h b/src/ldcache.h
+index 33d78dd..2b087db 100644
+--- a/src/ldcache.h
++++ b/src/ldcache.h
+@@ -50,6 +50,6 @@ void ldcache_init(struct ldcache *, struct error *, const char *);
+ int  ldcache_open(struct ldcache *);
+ int  ldcache_close(struct ldcache *);
+ int  ldcache_resolve(struct ldcache *, uint32_t, const char *, const char * const [],
+-    char *[], size_t, ldcache_select_fn, void *);
++    char *[], size_t, const char*);
+ 
+ #endif /* HEADER_LDCACHE_H */
+diff --git a/src/nvc_info.c b/src/nvc_info.c
+index cc96542..3fe7612 100644
+--- a/src/nvc_info.c
++++ b/src/nvc_info.c
+@@ -163,15 +163,13 @@ find_library_paths(struct error *err, struct nvc_driver_info *info, const char *
+         if (path_resolve_full(err, path, root, ldcache) < 0)
+                 return (-1);
+         ldcache_init(&ld, err, path);
+-        if (ldcache_open(&ld) < 0)
+-                return (-1);
+ 
+         info->nlibs = size;
+         info->libs = array_new(err, size);
+         if (info->libs == NULL)
+                 goto fail;
+         if (ldcache_resolve(&ld, LIB_ARCH, root, libs,
+-            info->libs, info->nlibs, select_libraries, info) < 0)
++            info->libs, info->nlibs, info->nvrm_version) < 0)
+                 goto fail;
+ 
+         info->nlibs32 = size;
+@@ -179,13 +177,11 @@ find_library_paths(struct error *err, struct nvc_driver_info *info, const char *
+         if (info->libs32 == NULL)
+                 goto fail;
+         if (ldcache_resolve(&ld, LIB32_ARCH, root, libs,
+-            info->libs32, info->nlibs32, select_libraries, info) < 0)
++            info->libs32, info->nlibs32, info->nvrm_version) < 0)
+                 goto fail;
+         rv = 0;
+ 
+  fail:
+-        if (ldcache_close(&ld) < 0)
+-                return (-1);
+         return (rv);
+ }
+ 
+@@ -199,7 +195,7 @@ find_binary_paths(struct error *err, struct nvc_driver_info *info, const char *r
+         char path[PATH_MAX];
+         int rv = -1;
+ 
+-        if ((env = secure_getenv("PATH")) == NULL) {
++        if ((env = "/run/nvidia-docker/bin:/run/nvidia-docker/extras/bin") == NULL) {
+                 error_setx(err, "environment variable PATH not found");
+                 return (-1);
+         }
+diff --git a/src/nvc_ldcache.c b/src/nvc_ldcache.c
+index d41a24d..65b7878 100644
+--- a/src/nvc_ldcache.c
++++ b/src/nvc_ldcache.c
+@@ -331,7 +331,7 @@ nvc_ldcache_update(struct nvc_context *ctx, const struct nvc_container *cnt)
+         if (validate_args(ctx, cnt != NULL) < 0)
+                 return (-1);
+ 
+-        argv = (char * []){cnt->cfg.ldconfig, cnt->cfg.libs_dir, cnt->cfg.libs32_dir, NULL};
++        argv = (char * []){cnt->cfg.ldconfig, "-f", "/tmp/ld.so.conf.nvidia-host", "-C", "/tmp/ld.so.cache.nvidia-host", cnt->cfg.libs_dir, cnt->cfg.libs32_dir, NULL};
+         if (*argv[0] == '@') {
+                 /*
+                  * We treat this path specially to be relative to the host filesystem.

pkgs/applications/virtualization/nvidia-docker/libnvc.nix

@@ -0,0 +1,53 @@
+{ stdenv, lib, fetchFromGitHub, libelf, libcap, libseccomp }:
+
+with lib; let
+
+  modp-ver = "396.51";
+
+  nvidia-modprobe = fetchFromGitHub {
+    owner = "NVIDIA";
+    repo = "nvidia-modprobe";
+    rev = modp-ver;
+    sha256 = "1fw2qwc84k64agw6fx2v0mjf88aggph9c6qhs4cv7l3gmflv8qbk";
+  };
+
+in stdenv.mkDerivation rec {
+  name = "libnvidia-container-${version}";
+  version = "1.0.0";
+
+  src = fetchFromGitHub {
+    owner = "NVIDIA";
+    repo = "libnvidia-container";
+    rev = "v${version}";
+    sha256 = "1ws6mfsbgxhzlb5w1r8qqg2arvxkr21n59i4cqsyz3h5jsqsflbw";
+  };
+
+  # locations of nvidia-driver libraries are not resolved via ldconfig which
+  # doesn't get used on NixOS. Additional support binaries like nvidia-smi are
+  # not resolved via the environment PATH but via the derivation output path.
+  patches = [ ./libnvc-ldconfig-and-path-fixes.patch ];
+
+  makeFlags = [
+    "WITH_LIBELF=yes"
+    "prefix=$(out)"
+  ];
+
+  postPatch = ''
+    sed -i 's/^REVISION :=.*/REVISION = ${src.rev}/' mk/common.mk
+    sed -i 's/^COMPILER :=.*/COMPILER = $(CC)/' mk/common.mk
+
+    mkdir -p deps/src/nvidia-modprobe-${modp-ver}
+    cp -r ${nvidia-modprobe}/* deps/src/nvidia-modprobe-${modp-ver}
+    chmod -R u+w deps/src
+    touch deps/src/nvidia-modprobe-${modp-ver}/.download_stamp
+  '';
+
+  buildInputs = [ libelf libcap libseccomp ];
+
+  meta = {
+    homepage = https://github.com/NVIDIA/libnvidia-container;
+    description = "NVIDIA container runtime library";
+    license = licenses.bsd3;
+    platforms = platforms.linux;
+  };
+}

pkgs/os-specific/linux/nvidia-x11/builder.sh

@@ -129,6 +129,8 @@ installPhase() {
         for i in nvidia-cuda-mps-control nvidia-cuda-mps-server nvidia-smi nvidia-debugdump; do
             if [ -e "$i" ]; then
                 install -Dm755 $i $bin/bin/$i
+                # unmodified binary backup for mounting in containers
+                install -Dm755 $i $bin/origBin/$i
                 patchelf --interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" \
                     --set-rpath $out/lib:$libPath $bin/bin/$i
             fi

pkgs/os-specific/linux/nvidia-x11/persistenced.nix

@@ -16,6 +16,11 @@ stdenv.mkDerivation rec {
   installFlags = [ "PREFIX=$(out)" ];
 
   postFixup = ''
+    # Save a copy of persistenced for mounting in containers
+    mkdir $out/origBin
+    cp $out/{bin,origBin}/nvidia-persistenced
+    patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 $out/origBin/nvidia-persistenced
+
     patchelf --set-rpath "$(patchelf --print-rpath $out/bin/nvidia-persistenced):${nvidia_x11}/lib" \
       $out/bin/nvidia-persistenced
   '';

pkgs/top-level/all-packages.nix

@@ -11839,6 +11839,8 @@ in
 
   nv-codec-headers = callPackage ../development/libraries/nv-codec-headers { };
 
+  nvidia-docker = callPackage ../applications/virtualization/nvidia-docker { };
+
   nvidia-texture-tools = callPackage ../development/libraries/nvidia-texture-tools { };
 
   nvidia-video-sdk = callPackage ../development/libraries/nvidia-video-sdk { };