Merge pull request #48803 from matthewbauer/crossOverlays

Adding pkgsStatic: a fully static overlay

pkgs/development/libraries/c-ares/default.nix

@@ -9,8 +9,6 @@ stdenv.mkDerivation rec {
     sha256 = "0lk8knip4xk6qzksdkn7085mmgm4ixfczdyyjw656c193y3rgnvc";
   };
 
-  configureFlags = stdenv.lib.optionals stdenv.hostPlatform.isWindows [ "--disable-shared" "--enable-static" ];
-
   meta = with stdenv.lib; {
     description = "A C library for asynchronous DNS requests";
     homepage = https://c-ares.haxx.se;

pkgs/development/libraries/ncurses/default.nix

@@ -4,6 +4,7 @@
 , mouseSupport ? false
 , unicode ? true
 , enableStatic ? stdenv.hostPlatform.useAndroidPrebuilt
+, enableShared ? !enableStatic
 , withCxx ? !stdenv.hostPlatform.useAndroidPrebuilt
 
 , gpm
@@ -29,7 +30,7 @@ stdenv.mkDerivation rec {
   setOutputFlags = false; # some aren't supported
 
   configureFlags = [
-    "--with-shared"
+    (lib.withFeature enableShared "shared")
     "--without-debug"
     "--enable-pc-files"
     "--enable-symlinks"

pkgs/development/libraries/zlib/default.nix

@@ -1,6 +1,7 @@
 { stdenv
 , fetchurl
-, static ? false
+, static ? true
+, shared ? true
 }:
 
 stdenv.mkDerivation (rec {
@@ -24,13 +25,15 @@ stdenv.mkDerivation (rec {
       --replace 'ARFLAGS="-o"' 'ARFLAGS="-r"'
   '';
 
-  outputs = [ "out" "dev" "static" ];
+  outputs = [ "out" "dev" ]
+    ++ stdenv.lib.optional (shared && static) "static";
   setOutputFlags = false;
   outputDoc = "dev"; # single tiny man3 page
 
-  configureFlags = stdenv.lib.optional (!static) "--shared";
+  configureFlags = stdenv.lib.optional shared "--shared"
+                   ++ stdenv.lib.optional (static && !shared) "--static";
 
-  postInstall = ''
+  postInstall = stdenv.lib.optionalString (shared && static) ''
     moveToOutput lib/libz.a "$static"
   ''
     # jww (2015-01-06): Sometimes this library install as a .so, even on
@@ -64,7 +67,7 @@ stdenv.mkDerivation (rec {
     "PREFIX=${stdenv.cc.targetPrefix}"
   ] ++ stdenv.lib.optionals (stdenv.hostPlatform.libc == "msvcrt") [
     "-f" "win32/Makefile.gcc"
-  ] ++ stdenv.lib.optionals (!static) [
+  ] ++ stdenv.lib.optionals shared [
     "SHARED_MODE=1"
   ];
 

pkgs/os-specific/darwin/apple-source-releases/libiconv/default.nix

@@ -15,7 +15,7 @@ appleDerivation {
     (lib.enableFeature enableShared "shared")
   ];
 
-  postInstall = lib.optionalString (!enableStatic) ''
+  postInstall = lib.optionalString enableShared ''
     mv $out/lib/libiconv.dylib $out/lib/libiconv-nocharset.dylib
     ${stdenv.cc.bintools.targetPrefix}install_name_tool -id $out/lib/libiconv-nocharset.dylib $out/lib/libiconv-nocharset.dylib
 

pkgs/stdenv/adapters.nix

@@ -32,13 +32,15 @@ rec {
   # Return a modified stdenv that tries to build statically linked
   # binaries.
   makeStaticBinaries = stdenv: stdenv //
-    { mkDerivation = args: stdenv.mkDerivation (args // {
-        NIX_CFLAGS_LINK = "-static";
+    { mkDerivation = args:
+      if stdenv.hostPlatform.isDarwin
+      then throw "Cannot build fully static binaries on Darwin/macOS"
+      else stdenv.mkDerivation (args // {
+        NIX_CFLAGS_LINK = toString (args.NIX_CFLAGS_LINK or "") + "-static";
         configureFlags = (args.configureFlags or []) ++ [
             "--disable-shared" # brrr...
           ];
       });
-      isStatic = true;
     };
 
 

pkgs/stdenv/cross/default.nix

@@ -1,11 +1,14 @@
 { lib
-, localSystem, crossSystem, config, overlays
+, localSystem, crossSystem, config, overlays, crossOverlays ? []
 }:
 
 let
   bootStages = import ../. {
     inherit lib localSystem overlays;
-    crossSystem = null;
+
+    crossSystem = localSystem;
+    crossOverlays = [];
+
     # Ignore custom stdenvs when cross compiling for compatability
     config = builtins.removeAttrs config [ "replaceStdenv" ];
   };
@@ -33,7 +36,8 @@ in lib.init bootStages ++ [
 
   # Run Packages
   (buildPackages: {
-    inherit config overlays;
+    inherit config;
+    overlays = overlays ++ crossOverlays;
     selfBuild = false;
     stdenv = buildPackages.stdenv.override (old: rec {
       buildPlatform = localSystem;
@@ -48,7 +52,7 @@ in lib.init bootStages ++ [
 
       cc = if crossSystem.useiOSPrebuilt or false
              then buildPackages.darwin.iosSdkPkgs.clang
-           else if crossSystem.useAndroidPrebuilt
+           else if crossSystem.useAndroidPrebuilt or false
              then buildPackages.androidenv."androidndkPkgs_${crossSystem.ndkVer}".gcc
            else buildPackages.gcc;
 
@@ -56,6 +60,7 @@ in lib.init bootStages ++ [
         ++ lib.optionals
              (hostPlatform.isLinux && !buildPlatform.isLinux)
              [ buildPackages.patchelf buildPackages.paxctl ]
+        ++ lib.optional hostPlatform.isDarwin buildPackages.clang
         ++ lib.optional
              (let f = p: !p.isx86 || p.libc == "musl"; in f hostPlatform && !(f buildPlatform))
              buildPackages.updateAutotoolsGnuConfigScriptsHook

pkgs/stdenv/custom/default.nix

@@ -2,7 +2,7 @@
 , localSystem, crossSystem, config, overlays
 }:
 
-assert crossSystem == null;
+assert crossSystem == localSystem;
 
 let
   bootStages = import ../. {

pkgs/stdenv/darwin/default.nix

@@ -1,5 +1,5 @@
 { lib
-, localSystem, crossSystem, config, overlays
+, localSystem, crossSystem, config, overlays, crossOverlays ? []
 
 # Allow passing in bootstrap files directly so we can test the stdenv bootstrap process when changing the bootstrap tools
 , bootstrapFiles ? let
@@ -16,7 +16,7 @@
   }
 }:
 
-assert crossSystem == null;
+assert crossSystem == localSystem;
 
 let
   inherit (localSystem) system platform;

pkgs/stdenv/darwin/portable-libsystem.sh

@@ -0,0 +1,10 @@
+# Make /nix/store/...-libSystem “portable” for static built binaries.
+# This just rewrites everything in $1/bin to use the
+# /usr/lib/libSystem.B.dylib that is provided on every macOS system.
+
+fixupOutputHooks+=('fixLibsystemRefs $prefix')
+
+fixLibsystemRefs() {
+  find "$1/bin" \
+    -exec install_name_tool -change @libsystem@ /usr/lib/libSystem.B.dylib {} \;
+}

pkgs/stdenv/default.nix

@@ -7,7 +7,7 @@
 { # Args just for stdenvs' usage
   lib
   # Args to pass on to the pkgset builder, too
-, localSystem, crossSystem, config, overlays
+, localSystem, crossSystem, config, overlays, crossOverlays ? []
 } @ args:
 
 let
@@ -36,7 +36,7 @@ let
 
   # Select the appropriate stages for the platform `system'.
 in
-  if crossSystem != null then stagesCross
+  if crossSystem != localSystem || crossOverlays != [] then stagesCross
   else if config ? replaceStdenv then stagesCustom
   else { # switch
     "i686-linux" = stagesLinux;

pkgs/stdenv/freebsd/default.nix

@@ -2,7 +2,7 @@
 , localSystem, crossSystem, config, overlays
 }:
 
-assert crossSystem == null;
+assert crossSystem == localSystem;
 let inherit (localSystem) system; in
 
 

pkgs/stdenv/linux/default.nix

@@ -4,7 +4,7 @@
 # compiler and linker that do not search in default locations,
 # ensuring purity of components produced by it.
 { lib
-, localSystem, crossSystem, config, overlays
+, localSystem, crossSystem, config, overlays, crossOverlays ? []
 
 , bootstrapFiles ?
   let table = {
@@ -32,7 +32,7 @@
   in files
 }:
 
-assert crossSystem == null;
+assert crossSystem == localSystem;
 
 let
   inherit (localSystem) system platform;

pkgs/stdenv/native/default.nix

@@ -2,7 +2,7 @@
 , localSystem, crossSystem, config, overlays
 }:
 
-assert crossSystem == null;
+assert crossSystem == localSystem;
 
 let
   inherit (localSystem) system;

pkgs/stdenv/nix/default.nix

@@ -1,10 +1,10 @@
 { lib
-, crossSystem, config, overlays
+, crossSystem, localSystem, config, overlays
 , bootStages
 , ...
 }:
 
-assert crossSystem == null;
+assert crossSystem == localSystem;
 
 bootStages ++ [
   (prevStage: {

pkgs/tools/system/proot/default.nix

@@ -1,6 +1,5 @@
 { stdenv, fetchFromGitHub, fetchpatch
-, talloc, docutils
-, enableStatic ? true }:
+, talloc, docutils }:
 
 ({ version, rev, sha256, patches }: stdenv.mkDerivation {
   name = "proot-${version}";
@@ -12,17 +11,13 @@
     owner = "cedric-vincent";
   };
 
-  buildInputs = [ talloc ] ++ stdenv.lib.optional enableStatic stdenv.cc.libc.static;
+  buildInputs = [ talloc ];
   nativeBuildInputs = [ docutils ];
 
   enableParallelBuilding = true;
 
   inherit patches;
 
-  preBuild = stdenv.lib.optionalString enableStatic ''
-    export LDFLAGS="-static"
-  '';
-
   makeFlags = [ "-C src" ];
 
   postBuild = ''

pkgs/top-level/all-packages.nix

@@ -3488,9 +3488,7 @@ in
     buildGoPackage = buildGo110Package;
   };
 
-  ipmitool = callPackage ../tools/system/ipmitool {
-    static = false;
-  };
+  ipmitool = callPackage ../tools/system/ipmitool { };
 
   ipmiutil = callPackage ../tools/system/ipmiutil {};
 
@@ -3983,8 +3981,6 @@ in
 
   libiberty = callPackage ../development/libraries/libiberty { };
 
-  libiberty_static = libiberty.override { staticBuild = true; };
-
   libxc = callPackage ../development/libraries/libxc { };
 
   libxcomp = callPackage ../development/libraries/libxcomp { };
@@ -4902,7 +4898,9 @@ in
   plowshare = callPackage ../tools/misc/plowshare { };
 
   pngcheck = callPackage ../tools/graphics/pngcheck {
-    zlib = zlibStatic;
+    zlib = zlib.override {
+      static = true;
+    };
   };
 
   pngcrush = callPackage ../tools/graphics/pngcrush { };
@@ -5210,7 +5208,7 @@ in
   routino = callPackage ../tools/misc/routino { };
 
   rq = callPackage ../development/tools/rq {
-    v8 = v8_static;
+    v8 = v8.override { static = true; };
   };
 
   rsnapshot = callPackage ../tools/backup/rsnapshot { };
@@ -8028,7 +8026,9 @@ in
   tcl-8_5 = callPackage ../development/interpreters/tcl/8.5.nix { };
   tcl-8_6 = callPackage ../development/interpreters/tcl/8.6.nix { };
 
-  proglodyte-wasm = callPackage ../development/interpreters/proglodyte-wasm { };
+  proglodyte-wasm = callPackage ../development/interpreters/proglodyte-wasm {
+    v8_static = v8.override { static = true; };
+  };
 
   wasm-gc = callPackage ../development/interpreters/wasm-gc { };
 
@@ -8450,7 +8450,9 @@ in
 
   lattice-diamond = callPackage ../development/tools/lattice-diamond { };
 
-  distcc = callPackage ../development/tools/misc/distcc { };
+  distcc = callPackage ../development/tools/misc/distcc {
+    libiberty_static = libiberty.override { staticBuild = true; };
+  };
 
   # distccWrapper: wrapper that works as gcc or g++
   # It can be used by setting in nixpkgs config like this, for example:
@@ -8811,7 +8813,9 @@ in
 
   openocd = callPackage ../development/tools/misc/openocd { };
 
-  oprofile = callPackage ../development/tools/profiling/oprofile { };
+  oprofile = callPackage ../development/tools/profiling/oprofile {
+    libiberty_static = libiberty.override { staticBuild = true; };
+  };
 
   pahole = callPackage ../development/tools/misc/pahole {};
 
@@ -12712,8 +12716,6 @@ in
     stdenv = overrideCC stdenv gcc6;
   });
 
-  v8_static = lowPrio (res.v8.override { static = true; });
-
   vaapiIntel = callPackage ../development/libraries/vaapi-intel { };
 
   vaapi-intel-hybrid = callPackage ../development/libraries/vaapi-intel-hybrid { };
@@ -12948,10 +12950,6 @@ in
 
   zlog = callPackage ../development/libraries/zlog { };
 
-  zlibStatic = lowPrio (appendToName "static" (zlib.override {
-    static = true;
-  }));
-
   zeromq3 = callPackage ../development/libraries/zeromq/3.x.nix {};
   zeromq4 = callPackage ../development/libraries/zeromq/4.x.nix {};
   zeromq = zeromq4;
@@ -20610,7 +20608,9 @@ in
   openspades = callPackage ../games/openspades { };
 
   openttd = callPackage ../games/openttd {
-    zlib = zlibStatic;
+    zlib = zlib.override {
+      static = true;
+    };
   };
 
   opentyrian = callPackage ../games/opentyrian { };

pkgs/top-level/default.nix

@@ -22,9 +22,8 @@
   # `*Platform`s.
   localSystem
 
-, # The system packages will ultimately be run on. Null if the two should be the
-  # same.
-  crossSystem ? null
+, # The system packages will ultimately be run on.
+  crossSystem ? localSystem
 
 , # Allow a configuration attribute set to be passed in as an argument.
   config ? {}
@@ -32,6 +31,9 @@
 , # List of overlays layers used to extend Nixpkgs.
   overlays ? []
 
+, # List of overlays to apply to target packages only.
+  crossOverlays ? []
+
 , # A function booting the final package set for a specific standard
   # environment. See below for the arguments given to that function, the type of
   # list it returns.
@@ -61,7 +63,8 @@ in let
     builtins.intersectAttrs { platform = null; } config
     // args.localSystem);
 
-  crossSystem = lib.mapNullable lib.systems.elaborate crossSystem0;
+  crossSystem = if crossSystem0 == null then localSystem
+                else lib.systems.elaborate crossSystem0;
 
   # A few packages make a new package set to draw their dependencies from.
   # (Currently to get a cross tool chain, or forced-i686 package.) Rather than
@@ -91,7 +94,7 @@ in let
   boot = import ../stdenv/booter.nix { inherit lib allPackages; };
 
   stages = stdenvStages {
-    inherit lib localSystem crossSystem config overlays;
+    inherit lib localSystem crossSystem config overlays crossOverlays;
   };
 
   pkgs = boot stages;

pkgs/top-level/stage.nix

@@ -174,6 +174,22 @@ let
     # Prefer appendOverlays if used repeatedly.
     extend = f: self.appendOverlays [f];
 
+    # Fully static packages.
+    # Currently uses Musl on Linux (couldn’t get static glibc to work).
+    pkgsStatic = nixpkgsFun ({
+      crossOverlays = [ (import ./static.nix) ];
+    } // lib.optionalAttrs stdenv.hostPlatform.isLinux {
+      crossSystem = {
+        parsed = stdenv.hostPlatform.parsed // {
+          abi = {
+            "gnu" = lib.systems.parse.abis.musl;
+            "gnueabi" = lib.systems.parse.abis.musleabi;
+            "gnueabihf" = lib.systems.parse.abis.musleabihf;
+          }.${stdenv.hostPlatform.parsed.abi.name}
+            or lib.systems.parse.abis.musl;
+        };
+      };
+    });
   };
 
   # The complete chain of package set builders, applied from top to bottom.

pkgs/top-level/static.nix

@@ -0,0 +1,150 @@
+# Overlay that builds static packages.
+
+# Not all packages will build but support is done on a
+# best effort basic.
+#
+# Note on Darwin/macOS: Apple does not provide a static libc
+# so any attempts at static binaries are going to be very
+# unsupported.
+#
+# Basic things like pkgsStatic.hello should work out of the box. More
+# complicated things will need to be fixed with overrides.
+
+self: super: let
+  inherit (super.stdenvAdapters) makeStaticBinaries
+                                 overrideInStdenv
+                                 makeStaticLibraries;
+  inherit (super.lib) foldl optional flip id optionalAttrs composeExtensions;
+  inherit (super) makeSetupHook;
+
+  # Best effort static binaries. Will still be linked to libSystem,
+  # but more portable than Nix store binaries.
+  makeStaticDarwin = stdenv: stdenv // {
+    mkDerivation = args: stdenv.mkDerivation (args // {
+      NIX_CFLAGS_LINK = toString (args.NIX_CFLAGS_LINK or "")
+                      + " -static-libgcc";
+      nativeBuildInputs = (args.nativeBuildInputs or []) ++ [ (makeSetupHook {
+        substitutions = {
+          libsystem = "${stdenv.cc.libc}/lib/libSystem.B.dylib";
+        };
+      } ../stdenv/darwin/portable-libsystem.sh) ];
+    });
+  };
+
+  staticAdapters = [ makeStaticLibraries ]
+
+    # Apple does not provide a static version of libSystem or crt0.o
+    # So we can’t build static binaries without extensive hacks.
+    ++ optional (!super.stdenv.hostPlatform.isDarwin) makeStaticBinaries
+
+    ++ optional super.stdenv.hostPlatform.isDarwin makeStaticDarwin
+
+    # Glibc doesn’t come with static runtimes by default.
+    # ++ optional (super.stdenv.hostPlatform.libc == "glibc") ((flip overrideInStdenv) [ self.stdenv.glibc.static ])
+  ;
+
+  # Force everything to link statically.
+  haskellStaticAdapter = self: super: {
+    mkDerivation = attrs: super.mkDerivation (attrs // {
+      enableSharedLibraries = false;
+      enableSharedExecutables = false;
+      enableStaticLibraries = true;
+    });
+  };
+
+in {
+  stdenv = foldl (flip id) super.stdenv staticAdapters;
+
+  haskell = super.haskell // {
+    packageOverrides = composeExtensions
+      (super.haskell.packageOverrides or (_: _: {}))
+      haskellStaticAdapter;
+  };
+
+  ncurses = super.ncurses.override {
+    enableStatic = true;
+  };
+  libxml2 = super.libxml2.override {
+    enableShared = false;
+    enableStatic = true;
+  };
+  zlib = super.zlib.override {
+    static = true;
+    shared = false;
+
+    # Don’t use new stdenv zlib because
+    # it doesn’t like the --disable-shared flag
+    stdenv = super.stdenv;
+  };
+  xz = super.xz.override {
+    enableStatic = true;
+  };
+  busybox = super.busybox.override {
+    enableStatic = true;
+  };
+  v8 = super.v8.override {
+    static = true;
+  };
+  libiberty = super.libiberty.override {
+    staticBuild = true;
+  };
+  ipmitool = super.ipmitool.override {
+    static = true;
+  };
+  neon = super.neon.override {
+    static = true;
+    shared = false;
+  };
+  libjpeg = super.libjpeg.override {
+    static = true;
+  };
+  gifsicle = super.gifsicle.override {
+    static = true;
+  };
+  bzip2 = super.bzip2.override {
+    linkStatic = true;
+  };
+  optipng = super.optipng.override {
+    static = true;
+  };
+  openssl = super.openssl.override {
+    static = true;
+
+    # Don’t use new stdenv for openssl because it doesn’t like the
+    # --disable-shared flag
+    stdenv = super.stdenv;
+  };
+  boost = super.boost.override {
+    enableStatic = true;
+    enableShared = false;
+  };
+  gmp = super.gmp.override {
+    withStatic = true;
+  };
+  cdo = super.cdo.override {
+    enable_all_static = true;
+  };
+  gsm = super.gsm.override {
+    staticSupport = true;
+  };
+  parted = super.parted.override {
+    enableStatic = true;
+  };
+  libiconvReal = super.libiconvReal.override {
+    enableShared = false;
+    enableStatic = true;
+  };
+  perl = super.perl.override {
+    # Don’t use new stdenv zlib because
+    # it doesn’t like the --disable-shared flag
+    stdenv = super.stdenv;
+  };
+
+  darwin = super.darwin // {
+    libiconv = super.darwin.libiconv.override {
+      enableShared = false;
+      enableStatic = true;
+    };
+  };
+
+}