From 0f316ecef2269f084545f137dd880147b5bd492b Mon Sep 17 00:00:00 2001 From: Pasquale Date: Sun, 28 Nov 2021 01:28:11 +0100 Subject: [PATCH 1/9] concatTextFile: init nixos/networking: using concatTextFile --- pkgs/build-support/trivial-builders.nix | 65 +++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/pkgs/build-support/trivial-builders.nix b/pkgs/build-support/trivial-builders.nix index 3c9f3189d2cf..ee5a492cdc18 100644 --- a/pkgs/build-support/trivial-builders.nix +++ b/pkgs/build-support/trivial-builders.nix @@ -322,6 +322,71 @@ rec { $CC -x c code.c -o "$n" ''; + + /* concat a list of files to the nix store. + * The contents of files are added to the file in the store. + * + * Examples: + * # Writes my-file to /nix/store/ + * concatTextFile { + * name = "my-file"; + * files = [ drv1 "${drv2}/path/to/file" ]; + * } + * # See also the `concatdText` helper function below. + * + * # Writes executable my-file to /nix/store//bin/my-file + * concatTextFile { + * name = "my-file"; + * files = '' + * Contents of File + * ''; + * executable = true; + * destination = "/bin/my-file"; + * } + */ + concatTextFile = + { name # the name of the derivation + , files + , executable ? false # run chmod +x ? + , destination ? "" # relative path appended to $out eg "/bin/foo" + , checkPhase ? "" # syntax checks, e.g. for scripts + , meta ? { } + }: + runCommandLocal name + { inherit files executable checkPhase meta destination; } + '' + n=$out$destination + mkdir -p "$(dirname "$n")" + cat $files > "$n" + touch "$n" + + eval "$checkPhase" + + (test -n "$executable" && chmod +x "$n") || true + ''; + + + /* + * Writes a text file to nix store with no optional parameters available. + * + * Example: + * # Writes contents of files to /nix/store/ + * concatText "my-file" [ file1 file2 ] + * + */ + concatText = name: files: concatTextFile { inherit name files; }; + + /* + * Writes a text file to nix store with and mark it as executable. + * + * Example: + * # Writes contents of files to /nix/store/ + * concatdScript "my-file" [ file1 file2 ] + * + */ + concatScript = name: files: concatTextFile { inherit name files; executable=true; }; + + /* * Create a forest of symlinks to the files in `paths'. * From f81734f4e69b0e74bcfc4e84a134387a5763d43c Mon Sep 17 00:00:00 2001 From: Pasquale Date: Sun, 28 Nov 2021 01:28:33 +0100 Subject: [PATCH 2/9] nixos/networking: using concatTextFile --- nixos/modules/config/networking.nix | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nixos/modules/config/networking.nix b/nixos/modules/config/networking.nix index 11307e331200..334f2e894900 100644 --- a/nixos/modules/config/networking.nix +++ b/nixos/modules/config/networking.nix @@ -190,9 +190,7 @@ in protocols.source = pkgs.iana-etc + "/etc/protocols"; # /etc/hosts: Hostname-to-IP mappings. - hosts.source = pkgs.runCommand "hosts" {} '' - cat ${escapeShellArgs cfg.hostFiles} > $out - ''; + hosts.source = pkgs.concatText "hosts" cfg.hostFiles; # /etc/netgroup: Network-wide groups. netgroup.text = mkDefault ""; From 67320a16689b84361777f3a03c8dcc5b72a339dc Mon Sep 17 00:00:00 2001 From: Pasquale Date: Sun, 28 Nov 2021 19:33:22 +0100 Subject: [PATCH 3/9] concatText: fix typos --- pkgs/build-support/trivial-builders.nix | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/pkgs/build-support/trivial-builders.nix b/pkgs/build-support/trivial-builders.nix index ee5a492cdc18..b02afb37943a 100644 --- a/pkgs/build-support/trivial-builders.nix +++ b/pkgs/build-support/trivial-builders.nix @@ -332,14 +332,12 @@ rec { * name = "my-file"; * files = [ drv1 "${drv2}/path/to/file" ]; * } - * # See also the `concatdText` helper function below. + * # See also the `concatText` helper function below. * * # Writes executable my-file to /nix/store//bin/my-file * concatTextFile { * name = "my-file"; - * files = '' - * Contents of File - * ''; + * files = [ drv1 "${drv2}/path/to/file" ]; * executable = true; * destination = "/bin/my-file"; * } @@ -355,14 +353,12 @@ rec { runCommandLocal name { inherit files executable checkPhase meta destination; } '' - n=$out$destination - mkdir -p "$(dirname "$n")" - cat $files > "$n" - touch "$n" + file=$out$destination + mkdir -p "$(dirname "$file")" + cat $files > "$file" + (test -n "$executable" && chmod +x "$file") || true eval "$checkPhase" - - (test -n "$executable" && chmod +x "$n") || true ''; @@ -381,7 +377,7 @@ rec { * * Example: * # Writes contents of files to /nix/store/ - * concatdScript "my-file" [ file1 file2 ] + * concatScript "my-file" [ file1 file2 ] * */ concatScript = name: files: concatTextFile { inherit name files; executable=true; }; From 767d56e8347c2ff2fb95a8dc4f5532ca148afba3 Mon Sep 17 00:00:00 2001 From: pasqui23 Date: Sun, 28 Nov 2021 20:43:17 +0000 Subject: [PATCH 4/9] concatScript: formatting Co-authored-by: Robert Hensing --- pkgs/build-support/trivial-builders.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/build-support/trivial-builders.nix b/pkgs/build-support/trivial-builders.nix index b02afb37943a..a7120e334333 100644 --- a/pkgs/build-support/trivial-builders.nix +++ b/pkgs/build-support/trivial-builders.nix @@ -380,7 +380,7 @@ rec { * concatScript "my-file" [ file1 file2 ] * */ - concatScript = name: files: concatTextFile { inherit name files; executable=true; }; + concatScript = name: files: concatTextFile { inherit name files; executable = true; }; /* From d2e237cbe19c70ad3c22956f51846eb0f729be7c Mon Sep 17 00:00:00 2001 From: Pasquale Date: Mon, 29 Nov 2021 18:59:33 +0100 Subject: [PATCH 5/9] concatText: documentation --- doc/builders/trivial-builders.chapter.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/builders/trivial-builders.chapter.md b/doc/builders/trivial-builders.chapter.md index c3a3572cd9f4..4a2f36ed2116 100644 --- a/doc/builders/trivial-builders.chapter.md +++ b/doc/builders/trivial-builders.chapter.md @@ -47,6 +47,11 @@ These functions write `text` to the Nix store. This is useful for creating scrip Many more commands wrap `writeTextFile` including `writeText`, `writeTextDir`, `writeScript`, and `writeScriptBin`. These are convenience functions over `writeTextFile`. +## `concatTextFile`, `concatText`, `concatScript` {#trivial-builder-concatText} + +These functions concatenate `files` to the Nix store in a single file. This is useful for configuration files structured in lines of text. `concatTextFile` takes an attribute set and expects two arguments, `name` and `files`. `name` corresponds to the name used in the Nix store path. `files` will be the files to be concatenated. You can also set `executable` to true to make this file have the executable bit set. +`concatText` and`concatScript` are simple wrappers over `concatTextFile`. + ## `writeShellApplication` {#trivial-builder-writeShellApplication} This can be used to easily produce a shell script that has some dependencies (`runtimeInputs`). It automatically sets the `PATH` of the script to contain all of the listed inputs, sets some sanity shellopts (`errexit`, `nounset`, `pipefail`), and checks the resulting script with [`shellcheck`](https://github.com/koalaman/shellcheck). From 1bc5179d1efa96979fbd6d283a2302da79d4a895 Mon Sep 17 00:00:00 2001 From: Pasquale Date: Thu, 2 Dec 2021 21:55:23 +0100 Subject: [PATCH 6/9] concatText: add test --- .../trivial-builders/test/concat-test.nix | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 pkgs/build-support/trivial-builders/test/concat-test.nix diff --git a/pkgs/build-support/trivial-builders/test/concat-test.nix b/pkgs/build-support/trivial-builders/test/concat-test.nix new file mode 100644 index 000000000000..d719ea7cd53c --- /dev/null +++ b/pkgs/build-support/trivial-builders/test/concat-test.nix @@ -0,0 +1,18 @@ +{ callPackage, lib, pkgs, runCommand, writeText, writeStringReferencesToFile }: +let + sample = import ./sample.nix { inherit pkgs; }; + samplePaths = lib.unique (lib.attrValues sample); + str2drv = x: "${x}"; + sampleText = concatText "cample-concat" (lib.unique (map str2drv samplePaths)); + stringReferencesText = + writeStringReferencesToFile + ((lib.concatMapStringsSep "fillertext" + stri + (lib.attrValues sample)) + '' + STORE=${builtins.storeDir};\nsystemctl start bar-foo.service + ''); +in +runCommand "test-writeStringReferencesToFile" { } '' + diff -U3 <(sort ${stringReferencesText}) <(sort ${sampleText}) + touch $out +'' From 6f0eae6d37f3b38e9dc1dde483d1bfa6738a85b2 Mon Sep 17 00:00:00 2001 From: Pasquale Date: Sat, 11 Dec 2021 19:50:03 +0100 Subject: [PATCH 7/9] docs: added examples to trivial builders section --- doc/builders/trivial-builders.chapter.md | 97 ++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/doc/builders/trivial-builders.chapter.md b/doc/builders/trivial-builders.chapter.md index 4a2f36ed2116..779a0a801b4e 100644 --- a/doc/builders/trivial-builders.chapter.md +++ b/doc/builders/trivial-builders.chapter.md @@ -47,11 +47,88 @@ These functions write `text` to the Nix store. This is useful for creating scrip Many more commands wrap `writeTextFile` including `writeText`, `writeTextDir`, `writeScript`, and `writeScriptBin`. These are convenience functions over `writeTextFile`. +Here are a few examples: +```nix +# Writes my-file to /nix/store/ +writeTextFile { + name = "my-file"; + text = '' + Contents of File + ''; +} +# See also the `writeText` helper function below. + +# Writes executable my-file to /nix/store//bin/my-file +writeTextFile { + name = "my-file"; + text = '' + Contents of File + ''; + executable = true; + destination = "/bin/my-file"; +} +# Writes contents of file to /nix/store/ +writeText "my-file" + '' + Contents of File + ''; +# Writes contents of file to /nix/store//share/my-file +writeTextDir "share/my-file" + '' + Contents of File + ''; +# Writes my-file to /nix/store/ and makes executable +writeScript "my-file" + '' + Contents of File + ''; +# Writes my-file to /nix/store//bin/my-file and makes executable. +writeScriptBin "my-file" + '' + Contents of File + ''; +# Writes my-file to /nix/store/ and makes executable. +writeShellScript "my-file" + '' + Contents of File + ''; +# Writes my-file to /nix/store//bin/my-file and makes executable. +writeShellScriptBin "my-file" + '' + Contents of File + ''; + +``` + ## `concatTextFile`, `concatText`, `concatScript` {#trivial-builder-concatText} These functions concatenate `files` to the Nix store in a single file. This is useful for configuration files structured in lines of text. `concatTextFile` takes an attribute set and expects two arguments, `name` and `files`. `name` corresponds to the name used in the Nix store path. `files` will be the files to be concatenated. You can also set `executable` to true to make this file have the executable bit set. `concatText` and`concatScript` are simple wrappers over `concatTextFile`. +Here are a few examples: +```nix + +# Writes my-file to /nix/store/ +concatTextFile { + name = "my-file"; + files = [ drv1 "${drv2}/path/to/file" ]; +} +# See also the `concatText` helper function below. + +# Writes executable my-file to /nix/store//bin/my-file +concatTextFile { + name = "my-file"; + files = [ drv1 "${drv2}/path/to/file" ]; + executable = true; + destination = "/bin/my-file"; +} +# Writes contents of files to /nix/store/ +concatText "my-file" [ file1 file2 ] + +# Writes contents of files to /nix/store/ +concatScript "my-file" [ file1 file2 ] +``` + ## `writeShellApplication` {#trivial-builder-writeShellApplication} This can be used to easily produce a shell script that has some dependencies (`runtimeInputs`). It automatically sets the `PATH` of the script to contain all of the listed inputs, sets some sanity shellopts (`errexit`, `nounset`, `pipefail`), and checks the resulting script with [`shellcheck`](https://github.com/koalaman/shellcheck). @@ -77,6 +154,26 @@ validation. ## `symlinkJoin` {#trivial-builder-symlinkJoin} This can be used to put many derivations into the same directory structure. It works by creating a new derivation and adding symlinks to each of the paths listed. It expects two arguments, `name`, and `paths`. `name` is the name used in the Nix store path for the created derivation. `paths` is a list of paths that will be symlinked. These paths can be to Nix store derivations or any other subdirectory contained within. +Here is an example: +```nix +# adds symlinks of hello and stack to current build and prints "links added" +symlinkJoin { name = "myexample"; paths = [ pkgs.hello pkgs.stack ]; postBuild = "echo links added"; } +``` +This creates a derivation with a directory structure like the following: +``` +/nix/store/sglsr5g079a5235hy29da3mq3hv8sjmm-myexample +|-- bin +| |-- hello -> /nix/store/qy93dp4a3rqyn2mz63fbxjg228hffwyw-hello-2.10/bin/hello +| `-- stack -> /nix/store/6lzdpxshx78281vy056lbk553ijsdr44-stack-2.1.3.1/bin/stack +`-- share + |-- bash-completion + | `-- completions + | `-- stack -> /nix/store/6lzdpxshx78281vy056lbk553ijsdr44-stack-2.1.3.1/share/bash-completion/completions/stack + |-- fish + | `-- vendor_completions.d + | `-- stack.fish -> /nix/store/6lzdpxshx78281vy056lbk553ijsdr44-stack-2.1.3.1/share/fish/vendor_completions.d/stack.fish +... +``` ## `writeReferencesToFile` {#trivial-builder-writeReferencesToFile} From 5187d2cd8f41b482dde3a7534143ccd526d67f08 Mon Sep 17 00:00:00 2001 From: Pasquale Date: Thu, 30 Dec 2021 13:12:34 +0100 Subject: [PATCH 8/9] concatText: test now works --- .../trivial-builders/test/concat-test.nix | 21 +++++++------------ pkgs/test/default.nix | 1 + 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/pkgs/build-support/trivial-builders/test/concat-test.nix b/pkgs/build-support/trivial-builders/test/concat-test.nix index d719ea7cd53c..59990a5124ab 100644 --- a/pkgs/build-support/trivial-builders/test/concat-test.nix +++ b/pkgs/build-support/trivial-builders/test/concat-test.nix @@ -1,18 +1,11 @@ -{ callPackage, lib, pkgs, runCommand, writeText, writeStringReferencesToFile }: +{ callPackage, lib, pkgs, runCommand, concatText, writeText, hello }: let - sample = import ./sample.nix { inherit pkgs; }; - samplePaths = lib.unique (lib.attrValues sample); - str2drv = x: "${x}"; - sampleText = concatText "cample-concat" (lib.unique (map str2drv samplePaths)); - stringReferencesText = - writeStringReferencesToFile - ((lib.concatMapStringsSep "fillertext" - stri - (lib.attrValues sample)) + '' - STORE=${builtins.storeDir};\nsystemctl start bar-foo.service - ''); + stri = writeText "pathToTest"; + txt1 = stri "abc"; + txt2 = stri hello; + res = concatText "textToTest" [ txt1 txt2 ]; in -runCommand "test-writeStringReferencesToFile" { } '' - diff -U3 <(sort ${stringReferencesText}) <(sort ${sampleText}) +runCommand "test-concatPaths" { } '' + diff -U3 <(cat ${txt1} ${txt2}) ${res} touch $out '' diff --git a/pkgs/test/default.nix b/pkgs/test/default.nix index e3ef7839c4b4..3f148eefef9c 100644 --- a/pkgs/test/default.nix +++ b/pkgs/test/default.nix @@ -57,6 +57,7 @@ with pkgs; writeStringReferencesToFile = callPackage ../build-support/trivial-builders/test/writeStringReferencesToFile.nix {}; references = callPackage ../build-support/trivial-builders/test/references.nix {}; overriding = callPackage ../build-support/trivial-builders/test-overriding.nix {}; + concat = callPackage ../build-support/trivial-builders/test/concat-test.nix {}; }; writers = callPackage ../build-support/writers/test.nix {}; From 86e2880d1146aa0df6025ad5d1da84e4a137317e Mon Sep 17 00:00:00 2001 From: Pasquale Date: Tue, 4 Jan 2022 23:14:50 +0100 Subject: [PATCH 9/9] tests.concat: added empty case --- pkgs/build-support/trivial-builders/test/concat-test.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgs/build-support/trivial-builders/test/concat-test.nix b/pkgs/build-support/trivial-builders/test/concat-test.nix index 59990a5124ab..5ce435619069 100644 --- a/pkgs/build-support/trivial-builders/test/concat-test.nix +++ b/pkgs/build-support/trivial-builders/test/concat-test.nix @@ -1,4 +1,4 @@ -{ callPackage, lib, pkgs, runCommand, concatText, writeText, hello }: +{ callPackage, lib, pkgs, runCommand, concatText, writeText, hello, emptyFile }: let stri = writeText "pathToTest"; txt1 = stri "abc"; @@ -7,5 +7,6 @@ let in runCommand "test-concatPaths" { } '' diff -U3 <(cat ${txt1} ${txt2}) ${res} + diff -U3 ${concatText "void" []} ${emptyFile} touch $out ''