cc-wrapper: simplify expandResponseParams parser

Import from b2446902fe

pkgs/build-support/cc-wrapper/default.nix

@@ -120,14 +120,14 @@ let
          null)
     else "";
 
-  parseResponseFile = if buildPackages.stdenv.cc or null != null && buildPackages.stdenv.cc != "/dev/null"
+  expand-response-params = if buildPackages.stdenv.cc or null != null && buildPackages.stdenv.cc != "/dev/null"
   then buildPackages.stdenv.mkDerivation {
-    name = "parse-response-file";
-    src = ./parseResponseFile.c;
+    name = "expand-response-params";
+    src = ./expand-response-params.c;
     buildCommand = ''
-      # Make sure the output file doesn't refer to the input nix path
-      cp "$src" parseResponseFile.c
-      "$CC" -O3 -o "$out" parseResponseFile.c
+      # Work around "stdenv-darwin-boot-2 is not allowed to refer to path /nix/store/...-expand-response-params.c"
+      cp "$src" expand-response-params.c
+      "$CC" -std=c99 -O3 -o "$out" expand-response-params.c
     '';
   } else "";
 
@@ -383,7 +383,9 @@ stdenv.mkDerivation {
     ''
     + extraBuildCommands;
 
-  inherit dynamicLinker parseResponseFile;
+  inherit dynamicLinker expand-response-params;
+
+  expandResponseParams = expand-response-params; # for substitution in utils.sh
 
   crossAttrs = {
     shell = shell.crossDrv + shell.crossDrv.shellPath;

pkgs/build-support/cc-wrapper/expand-response-params.c

@@ -0,0 +1,84 @@
+#include <assert.h>
+#include <ctype.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+typedef struct { char *data; size_t len, cap; } String;
+
+void resize(String *s, size_t len) {
+    s->len = len;
+    if (s->cap < s->len) {
+        s->cap = s->len * 2;
+        s->data = (char *)realloc(s->data, s->cap);
+        assert(s->data);
+    }
+}
+
+void append(String *s, const char *data, size_t len) {
+    resize(s, s->len + len);
+    memcpy(s->data + s->len - len, data, len);
+}
+
+typedef enum { space = 0, other = 1, backslash = 2, apostrophe = 3, quotation_mark = 4 } CharClass;
+typedef enum { outside, unq, unq_esc, sq, sq_esc, dq, dq_esc } State;
+
+// current State -> CharClass -> next State
+const State transitions[][5] = {
+    [outside] = {outside, unq, unq_esc, sq,  dq},
+    [unq]     = {outside, unq, unq_esc, sq,  dq},
+    [unq_esc] = {unq,     unq, unq,     unq, unq},
+    [sq]      = {sq,      sq,  sq_esc,  unq, sq},
+    [sq_esc]  = {sq,      sq,  sq,      sq,  sq},
+    [dq]      = {dq,      dq,  dq_esc,  dq,  unq},
+    [dq_esc]  = {dq,      dq,  dq,      dq,  dq},
+};
+
+CharClass charClass(int c) {
+    return c == '\\' ? backslash : c == '\'' ? apostrophe : c == '"' ? quotation_mark :
+            isspace(c) ? space : other;
+}
+
+// expandArg writes NULL-terminated expansions of `arg', a NULL-terminated
+// string, to stdout.  If arg does not begin with `@' or does not refer to a
+// file, it is written as is.  Otherwise the contents of the file are
+// recursively expanded.  On unexpected EOF in malformed response files an
+// incomplete final argument is written, even if it is empty, to parse like GCC.
+void expandArg(String *arg) {
+    FILE *f;
+    if (arg->data[0] != '@' || !(f = fopen(&arg->data[1], "r"))) {
+        fwrite(arg->data, 1, arg->len, stdout);
+        return;
+    }
+
+    resize(arg, 0);
+    State cur = outside;
+    int c;
+    do {
+        c = fgetc(f);
+        State next = transitions[cur][charClass(c)];
+        if ((cur == unq && next == outside) || (cur != outside && c == EOF)) {
+            append(arg, "", 1);
+            expandArg(arg);
+            resize(arg, 0);
+        } else if (cur == unq_esc || cur == sq_esc || cur == dq_esc ||
+                   cur == outside ? next == unq : cur == next) {
+            char s = c;
+            append(arg, &s, 1);
+        }
+        cur = next;
+    } while (c != EOF);
+
+    fclose(f);
+}
+
+int main(int argc, char **argv) {
+    String arg = { 0 };
+    while (*++argv) {
+        resize(&arg, 0);
+        append(&arg, *argv, strlen(*argv) + 1);
+        expandArg(&arg);
+    }
+    free(arg.data);
+    return EXIT_SUCCESS;
+}

pkgs/build-support/cc-wrapper/parseResponseFile.c

@@ -1,181 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-
-typedef enum {
-  UNQUOTED,
-  SINGLE_QUOTED,
-  DOUBLE_QUOTED
-} quote_state;
-
-typedef enum {
-  BEFORE_FIRST_WORD,
-  IN_WORD,
-  AFTER_WORD
-} word_break_state;
-
-void emitWordChar(word_break_state *w, char c) {
-  switch(*w) { // Note: These all fall through
-  case AFTER_WORD:
-    putchar(' ');
-  case BEFORE_FIRST_WORD:
-    putchar('\'');
-    *w = IN_WORD;
-  case IN_WORD:
-    if(c == '\'') {
-      printf("'\\''");
-    } else {
-      putchar(c);
-    }
-  }
-}
-
-void emitWordEnd(word_break_state *w) {
-  if(*w == IN_WORD) {
-    putchar('\'');
-    *w = AFTER_WORD;
-  } // Otherwise, the state remains the same
-}
-
-typedef struct {
-  word_break_state *w;
-  char *subFilename; // Non-null if we're currently accumulating a response file name
-  size_t subFilenameUsed; // If subFilename == 0, this should be 0; this should always be less than (subFilenameAllocated - 1), to allow room for the null byte
-  size_t subFilenameAllocated; // If subFilename == 0, this should be 0
-} file_state; // The state of parsing a single file
-
-static const unsigned int INITIAL_SUB_FILENAME_CHARS = 32; // Arbitrary, but must be > 0
-
-void *exitIfNull(void *p) {
-  if(!p) {
-    fprintf(stderr, "Out of memory");
-    exit(2);
-  }
-  return p;
-}
-
-void wordChar(file_state *s, char c) {
-  if(s->subFilename) { // We're accumulating a file to recursively process
-    // Allocate more space if we need to
-    if(s->subFilenameUsed >= s->subFilenameAllocated - 1) {
-      size_t newSize = s->subFilenameAllocated * 2;
-      s->subFilename = exitIfNull(realloc(s->subFilename, newSize));
-      s->subFilenameAllocated = newSize;
-    }
-    s->subFilename[s->subFilenameUsed++] = c;
-  } else if(*s->w != IN_WORD && c == '@') { // This is the first letter in the word; note that even quoted or escaped @'s are recursively interpreted
-    s->subFilename = exitIfNull(malloc(INITIAL_SUB_FILENAME_CHARS * sizeof(*(s->subFilename))));
-    assert(s->subFilenameUsed == 0);
-    assert(s->subFilenameAllocated == 0);
-    s->subFilenameAllocated = INITIAL_SUB_FILENAME_CHARS;
-  } else {
-    emitWordChar(s->w, c);
-  }
-}
-
-void processFile(word_break_state *w, const char *filename);
-
-void endWord(file_state *s) {
-  if(s->subFilename) {
-    s->subFilename[s->subFilenameUsed] = '\0';
-
-    processFile(s->w, s->subFilename);
-
-    free(s->subFilename);
-    s->subFilename = 0;
-    s->subFilenameUsed = 0;
-    s->subFilenameAllocated = 0;
-  } else {
-    emitWordEnd(s->w);
-  }
-}
-
-void processFile(word_break_state *w, const char *filename) {
-  FILE *h = fopen(filename, "r");
-  if(!h) { //TODO: We assume it's because the file doesn't exist, but perhaps we should check for other failure cases
-    emitWordChar(w, '@');
-    while(*filename) {
-      emitWordChar(w, *filename);
-      ++filename;
-    }
-    emitWordEnd(w);
-    return;
-  }
-
-  char c;
-  quote_state q = UNQUOTED;
-  file_state s = {
-    .w = w,
-    .subFilename = 0,
-    .subFilenameUsed = 0,
-    .subFilenameAllocated = 0
-  };
-  while((c = fgetc(h)) != EOF) {
-    //fprintf(stderr, "%d\n", c);
-    switch(c) {
-    case '\'':
-      switch(q) {
-      case UNQUOTED:
-        q = SINGLE_QUOTED;
-        break;
-      case SINGLE_QUOTED:
-        q = UNQUOTED;
-        break;
-      case DOUBLE_QUOTED:
-        wordChar(&s, '\'');
-        break;
-      }
-      break;
-    case '"':
-      switch(q) {
-      case UNQUOTED:
-        q = DOUBLE_QUOTED;
-        break;
-      case SINGLE_QUOTED:
-        wordChar(&s, '"');
-        break;
-      case DOUBLE_QUOTED:
-        q = UNQUOTED;
-        break;
-      }
-      break;
-    case '\\':
-      c = fgetc(h);
-      if(c != EOF) {
-        wordChar(&s, c);
-      }
-      break;
-    case ' ':
-    case '\t':
-    case '\n':
-    case '\v':
-    case '\f':
-    case '\r':
-      if(q == UNQUOTED) {
-        endWord(&s);
-      } else {
-        wordChar(&s, c);
-      }
-      break;
-    default:
-      wordChar(&s, c);
-      break;
-    }
-  }
-
-  endWord(&s);
-
-  fclose(h);
-}
-
-int main(int argc, const char *argv[]) {
-  if(argc != 2) {
-    fprintf(stderr, "Usage: %s [responsefile]", argv[0]);
-    return 1;
-  }
-
-  word_break_state w = BEFORE_FIRST_WORD;
-  processFile(&w, argv[1]);
-
-  return 0;
-}

pkgs/build-support/cc-wrapper/utils.sh

@@ -24,26 +24,17 @@ badPath() {
 }
 
 expandResponseParams() {
-    local inparams=("$@")
-    local n=0
-    local p
-    params=()
-    while [ $n -lt ${#inparams[*]} ]; do
-        p=${inparams[n]}
-        case $p in
-            @*)
-                local parseResponseFile="@parseResponseFile@"
-                if [ -n "$parseResponseFile" ] ; then
-                    eval "params+=($("$parseResponseFile" "${p:1}"))"
-                else
-                    echo "Response files aren't supported during bootstrapping" >&2
-                    exit 1
-                fi
-                ;;
-            *)
-                params+=("$p")
-                ;;
-        esac
-        n=$((n + 1))
+    params=("$@")
+    local arg
+    for arg in "$@"; do
+        if [[ "$arg" == @* ]]; then
+            if [ -n "@expandResponseParams@" ]; then
+                readarray -d '' params < <("@expandResponseParams@" "$@")
+                return 0
+            else
+                echo "Response files aren't supported during bootstrapping" >&2
+                return 1
+            fi
+        fi
     done
 }

pkgs/stdenv/darwin/default.nix

@@ -54,7 +54,7 @@ in rec {
     __sandboxProfile = binShClosure + libSystemProfile;
   };
 
-  stageFun = step: last: {shell             ? "${bootstrapTools}/bin/sh",
+  stageFun = step: last: {shell             ? "${bootstrapTools}/bin/bash",
                           overrides         ? (self: super: {}),
                           extraPreHook      ? "",
                           extraBuildInputs,
@@ -63,7 +63,7 @@ in rec {
       thisStdenv = import ../generic {
         inherit config shell extraBuildInputs;
         allowedRequisites = if allowedRequisites == null then null else allowedRequisites ++ [
-          thisStdenv.cc.parseResponseFile
+          thisStdenv.cc.expand-response-params
         ];
 
         name = "stdenv-darwin-boot-${toString step}";
@@ -86,7 +86,7 @@ in rec {
           cc           = { name = "clang-9.9.9"; outPath = bootstrapTools; };
         };
 
-        preHook = stage0.stdenv.lib.optionalString (shell == "${bootstrapTools}/bin/sh") ''
+        preHook = stage0.stdenv.lib.optionalString (shell == "${bootstrapTools}/bin/bash") ''
           # Don't patch #!/interpreter because it leads to retained
           # dependencies on the bootstrapTools in the final stdenv.
           dontPatchShebangs=1
@@ -328,7 +328,7 @@ in rec {
       gzip ncurses.out ncurses.dev ncurses.man gnused bash gawk
       gnugrep llvmPackages.clang-unwrapped patch pcre.out binutils-raw.out
       binutils-raw.dev binutils gettext
-      cc.parseResponseFile
+      cc.expand-response-params
     ]) ++ (with pkgs.darwin; [
       dyld Libsystem CF cctools ICU libiconv locale
     ]);