Merge pull request #26974 from obsidiansystems/response-file-parsing-speed

cc-wrapper: improve response file parsing speed

pkgs/build-support/cc-wrapper/default.nix

@@ -10,7 +10,7 @@
 , zlib ? null, extraPackages ? [], extraBuildCommands ? ""
 , dyld ? null # TODO: should this be a setup-hook on dyld?
 , isGNU ? false, isClang ? cc.isClang or false, gnugrep ? null
-, hostPlatform, targetPlatform
+, buildPackages ? {}, hostPlatform, targetPlatform
 , runCommand ? null
 }:
 
@@ -120,6 +120,17 @@ let
          null)
     else "";
 
+  expand-response-params = if buildPackages.stdenv.cc or null != null && buildPackages.stdenv.cc != "/dev/null"
+  then buildPackages.stdenv.mkDerivation {
+    name = "expand-response-params";
+    src = ./expand-response-params.c;
+    buildCommand = ''
+      # Work around "stdenv-darwin-boot-2 is not allowed to refer to path /nix/store/...-expand-response-params.c"
+      cp "$src" expand-response-params.c
+      "$CC" -std=c99 -O3 -o "$out" expand-response-params.c
+    '';
+  } else "";
+
 in
 
 stdenv.mkDerivation {
@@ -368,11 +379,13 @@ stdenv.mkDerivation {
     + ''
       substituteAll ${preWrap ./add-flags.sh} $out/nix-support/add-flags.sh
       substituteAll ${preWrap ./add-hardening.sh} $out/nix-support/add-hardening.sh
-      cp -p ${preWrap ./utils.sh} $out/nix-support/utils.sh
+      substituteAll ${preWrap ./utils.sh} $out/nix-support/utils.sh
     ''
     + extraBuildCommands;
 
-  inherit dynamicLinker;
+  inherit dynamicLinker expand-response-params;
+
+  expandResponseParams = expand-response-params; # for substitution in utils.sh
 
   crossAttrs = {
     shell = shell.crossDrv + shell.crossDrv.shellPath;

pkgs/build-support/cc-wrapper/expand-response-params.c

@@ -0,0 +1,84 @@
+#include <assert.h>
+#include <ctype.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+typedef struct { char *data; size_t len, cap; } String;
+
+void resize(String *s, size_t len) {
+    s->len = len;
+    if (s->cap < s->len) {
+        s->cap = s->len * 2;
+        s->data = (char *)realloc(s->data, s->cap);
+        assert(s->data);
+    }
+}
+
+void append(String *s, const char *data, size_t len) {
+    resize(s, s->len + len);
+    memcpy(s->data + s->len - len, data, len);
+}
+
+typedef enum { space = 0, other = 1, backslash = 2, apostrophe = 3, quotation_mark = 4 } CharClass;
+typedef enum { outside, unq, unq_esc, sq, sq_esc, dq, dq_esc } State;
+
+// current State -> CharClass -> next State
+const State transitions[][5] = {
+    [outside] = {outside, unq, unq_esc, sq,  dq},
+    [unq]     = {outside, unq, unq_esc, sq,  dq},
+    [unq_esc] = {unq,     unq, unq,     unq, unq},
+    [sq]      = {sq,      sq,  sq_esc,  unq, sq},
+    [sq_esc]  = {sq,      sq,  sq,      sq,  sq},
+    [dq]      = {dq,      dq,  dq_esc,  dq,  unq},
+    [dq_esc]  = {dq,      dq,  dq,      dq,  dq},
+};
+
+CharClass charClass(int c) {
+    return c == '\\' ? backslash : c == '\'' ? apostrophe : c == '"' ? quotation_mark :
+            isspace(c) ? space : other;
+}
+
+// expandArg writes NULL-terminated expansions of `arg', a NULL-terminated
+// string, to stdout.  If arg does not begin with `@' or does not refer to a
+// file, it is written as is.  Otherwise the contents of the file are
+// recursively expanded.  On unexpected EOF in malformed response files an
+// incomplete final argument is written, even if it is empty, to parse like GCC.
+void expandArg(String *arg) {
+    FILE *f;
+    if (arg->data[0] != '@' || !(f = fopen(&arg->data[1], "r"))) {
+        fwrite(arg->data, 1, arg->len, stdout);
+        return;
+    }
+
+    resize(arg, 0);
+    State cur = outside;
+    int c;
+    do {
+        c = fgetc(f);
+        State next = transitions[cur][charClass(c)];
+        if ((cur == unq && next == outside) || (cur != outside && c == EOF)) {
+            append(arg, "", 1);
+            expandArg(arg);
+            resize(arg, 0);
+        } else if (cur == unq_esc || cur == sq_esc || cur == dq_esc ||
+                   (cur == outside ? next == unq : cur == next)) {
+            char s = c;
+            append(arg, &s, 1);
+        }
+        cur = next;
+    } while (c != EOF);
+
+    fclose(f);
+}
+
+int main(int argc, char **argv) {
+    String arg = { 0 };
+    while (*++argv) {
+        resize(&arg, 0);
+        append(&arg, *argv, strlen(*argv) + 1);
+        expandArg(&arg);
+    }
+    free(arg.data);
+    return EXIT_SUCCESS;
+}

pkgs/build-support/cc-wrapper/utils.sh

@@ -23,52 +23,18 @@ badPath() {
         "${p:0:${#NIX_BUILD_TOP}}" != "$NIX_BUILD_TOP"
 }
 
-# @args.rsp parser.
-# Char classes: space, other, backslash, single quote, double quote.
-# States: 0 - outside, 1/2 - unquoted arg/slash, 3/4 - 'arg'/slash, 5/6 - "arg"/slash.
-# State transitions:
-rspT=(01235 01235 11111 33413 33333 55651 55555)
-# Push (a) arg or (c) char on transition:
-rspP[10]=a rspP[01]=c rspP[11]=c rspP[21]=c rspP[33]=c rspP[43]=c rspP[55]=c rspP[65]=c
-
-rspParse() {
-    rsp=()
-    local state=0
-    local arg=''
-    local c
-
-    while read -r -N1 c; do
-        local cls=1
-        case "$c" in
-            ' ' | $'\t' | $'\r' | $'\n') cls=0 ;;
-            '\') cls=2 ;;
-            "'") cls=3 ;;
-            '"') cls=4 ;;
-        esac
-        local nextstates="${rspT[$state]}"
-        local nextstate="${nextstates:$cls:1}"
-        case "${rspP[$state$nextstate]}" in
-            'c') arg+="$c" ;;
-            'a') rsp+=("$arg"); arg='' ;;
-        esac
-        state="$nextstate"
-    done
-
-    if [ "$state" -ne 0 ]; then
-        rsp+=("$arg")
-    fi
-}
-
 expandResponseParams() {
-    params=()
-    while [ $# -gt 0 ]; do
-        local p="$1"
-        shift
-        if [ "${p:0:1}" = '@' -a -e "${p:1}" ]; then
-            rspParse <"${p:1}"
-            set -- "${rsp[@]}" "$@"
-        else
-            params+=("$p")
+    params=("$@")
+    local arg
+    for arg in "$@"; do
+        if [[ "$arg" == @* ]]; then
+            if [ -n "@expandResponseParams@" ]; then
+                readarray -d '' params < <("@expandResponseParams@" "$@")
+                return 0
+            else
+                echo "Response files aren't supported during bootstrapping" >&2
+                return 1
+            fi
         fi
     done
 }

pkgs/os-specific/darwin/ios-cross/default.nix

@@ -6,7 +6,9 @@
 , stdenv
 , coreutils
 , gnugrep
-, hostPlatform, targetPlatform
+, buildPackages
+, hostPlatform
+, targetPlatform
 }:
 
 /* As of this writing, known-good prefix/arch/simulator triples:
@@ -29,7 +31,7 @@ let
   sdk = "/Applications/Xcode.app/Contents/Developer/Platforms/iPhone${sdkType}.platform/Developer/SDKs/iPhone${sdkType}${sdkVer}.sdk";
 
 in (import ../../../build-support/cc-wrapper {
-    inherit stdenv coreutils gnugrep runCommand;
+    inherit stdenv coreutils gnugrep runCommand buildPackages;
     nativeTools = false;
     nativeLibc = false;
     inherit binutils;

pkgs/stdenv/darwin/default.nix

@@ -54,14 +54,17 @@ in rec {
     __sandboxProfile = binShClosure + libSystemProfile;
   };
 
-  stageFun = step: last: {shell             ? "${bootstrapTools}/bin/sh",
+  stageFun = step: last: {shell             ? "${bootstrapTools}/bin/bash",
                           overrides         ? (self: super: {}),
                           extraPreHook      ? "",
                           extraBuildInputs,
                           allowedRequisites ? null}:
     let
       thisStdenv = import ../generic {
-        inherit config shell extraBuildInputs allowedRequisites;
+        inherit config shell extraBuildInputs;
+        allowedRequisites = if allowedRequisites == null then null else allowedRequisites ++ [
+          thisStdenv.cc.expand-response-params
+        ];
 
         name = "stdenv-darwin-boot-${toString step}";
 
@@ -73,6 +76,9 @@ in rec {
           nativeTools  = true;
           nativePrefix = bootstrapTools;
           nativeLibc   = false;
+          buildPackages = lib.optionalAttrs (last ? stdenv) {
+            inherit (last) stdenv;
+          };
           hostPlatform = localSystem;
           targetPlatform = localSystem;
           libc         = last.pkgs.darwin.Libsystem;
@@ -80,7 +86,7 @@ in rec {
           cc           = { name = "clang-9.9.9"; outPath = bootstrapTools; };
         };
 
-        preHook = stage0.stdenv.lib.optionalString (shell == "${bootstrapTools}/bin/sh") ''
+        preHook = stage0.stdenv.lib.optionalString (shell == "${bootstrapTools}/bin/bash") ''
           # Don't patch #!/interpreter because it leads to retained
           # dependencies on the bootstrapTools in the final stdenv.
           dontPatchShebangs=1
@@ -297,6 +303,9 @@ in rec {
       inherit shell;
       nativeTools = false;
       nativeLibc  = false;
+      buildPackages = {
+        inherit (prevStage) stdenv;
+      };
       hostPlatform = localSystem;
       targetPlatform = localSystem;
       inherit (pkgs) coreutils binutils gnugrep;
@@ -319,6 +328,7 @@ in rec {
       gzip ncurses.out ncurses.dev ncurses.man gnused bash gawk
       gnugrep llvmPackages.clang-unwrapped patch pcre.out binutils-raw.out
       binutils-raw.dev binutils gettext
+      cc.expand-response-params
     ]) ++ (with pkgs.darwin; [
       dyld Libsystem CF cctools ICU libiconv locale
     ]);

pkgs/stdenv/linux/default.nix

@@ -76,6 +76,9 @@ let
              else lib.makeOverridable (import ../../build-support/cc-wrapper) {
           nativeTools = false;
           nativeLibc = false;
+          buildPackages = lib.optionalAttrs (prevStage ? stdenv) {
+            inherit (prevStage) stdenv;
+          };
           hostPlatform = localSystem;
           targetPlatform = localSystem;
           cc = prevStage.gcc-unwrapped;
@@ -241,6 +244,9 @@ in
         nativeTools = false;
         nativeLibc = false;
         isGNU = true;
+        buildPackages = {
+          inherit (prevStage) stdenv;
+        };
         hostPlatform = localSystem;
         targetPlatform = localSystem;
         cc = prevStage.gcc-unwrapped;