From c154dd5101c816f66d4c6e992d8f9a667e7e45c4 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Fri, 17 Jan 2020 16:03:00 +0100 Subject: [PATCH 1/7] nixos/buildkite-agent: stop deploying public keys, make ssh private key optional. SSH public keys aren't needed to clone private repos, and if we only need to configure a single attribute, there's no need for the "openssh" attrset anymore. --- .../buildkite-agent.nix | 38 ++++++++----------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/nixos/modules/services/continuous-integration/buildkite-agent.nix b/nixos/modules/services/continuous-integration/buildkite-agent.nix index e996680bedaf..49fa4271caca 100644 --- a/nixos/modules/services/continuous-integration/buildkite-agent.nix +++ b/nixos/modules/services/continuous-integration/buildkite-agent.nix @@ -93,26 +93,19 @@ in ''; }; - openssh = - { privateKeyPath = mkOption { - type = types.path; - description = '' - Private agent key. + privateSshKeyPath = mkOption { + type = types.path; + ## maximum care is taken so that secrets (ssh keys and the CI token) + ## don't end up in the Nix store. + apply = final: if final == null then null else toString final; - A run-time path to the key file, which is supposed to be provisioned - outside of Nix store. - ''; - }; - publicKeyPath = mkOption { - type = types.path; - description = '' - Public agent key. + description = '' + OpenSSH private key - A run-time path to the key file, which is supposed to be provisioned - outside of Nix store. - ''; - }; - }; + A run-time path to the key file, which is supposed to be provisioned + outside of Nix store. + ''; + }; hooks = mkHookOptions [ { name = "checkout"; @@ -217,7 +210,6 @@ in '' mkdir -m 0700 -p "${sshDir}" cp -f "${toString cfg.openssh.privateKeyPath}" "${sshDir}/id_rsa" - cp -f "${toString cfg.openssh.publicKeyPath}" "${sshDir}/id_rsa.pub" chmod 600 "${sshDir}"/id_rsa* cat > "${cfg.dataDir}/buildkite-agent.cfg" < Date: Fri, 17 Jan 2020 16:28:31 +0100 Subject: [PATCH 2/7] nixos/buildkite-agent: actually use services.buildkite-agent.package We were currently just using pkgs.buildkite-agent, no matter what was configured in services.buildkite-agent.package --- .../modules/services/continuous-integration/buildkite-agent.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/services/continuous-integration/buildkite-agent.nix b/nixos/modules/services/continuous-integration/buildkite-agent.nix index 49fa4271caca..ca31865b9c29 100644 --- a/nixos/modules/services/continuous-integration/buildkite-agent.nix +++ b/nixos/modules/services/continuous-integration/buildkite-agent.nix @@ -223,7 +223,7 @@ in ''; serviceConfig = - { ExecStart = "${cfg.buildkite-agent}/bin/buildkite-agent start --config /var/lib/buildkite-agent/buildkite-agent.cfg"; + { ExecStart = "${cfg.package}/bin/buildkite-agent start --config /var/lib/buildkite-agent/buildkite-agent.cfg"; User = "buildkite-agent"; RestartSec = 5; Restart = "on-failure"; From 99d0f4c38064841dcc3a188667099e491ebc9932 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Fri, 17 Jan 2020 16:47:34 +0100 Subject: [PATCH 3/7] nixos/buildkite-agent: tweak TimeoutStopSec and KillMode This improves behaviour when the service is being stopped. --- .../services/continuous-integration/buildkite-agent.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nixos/modules/services/continuous-integration/buildkite-agent.nix b/nixos/modules/services/continuous-integration/buildkite-agent.nix index ca31865b9c29..3da1e5e79e21 100644 --- a/nixos/modules/services/continuous-integration/buildkite-agent.nix +++ b/nixos/modules/services/continuous-integration/buildkite-agent.nix @@ -228,6 +228,9 @@ in RestartSec = 5; Restart = "on-failure"; TimeoutSec = 10; + # set a long timeout to give buildkite-agent a chance to finish current builds + TimeoutStopSec = "2 min"; + KillMode = "mixed"; }; }; From 8f19d7211313df46c30ab8b0748a6924e00c6874 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Wed, 26 Sep 2018 19:57:32 +0200 Subject: [PATCH 4/7] buildkite-agent2: remove package The latest 2.x release was in Jan 2018. --- .../buildkite-agent/2.x.nix | 12 ----- .../buildkite-agent/3.x.nix | 15 ------ .../buildkite-agent/default.nix | 48 +++++++++++++++++++ pkgs/top-level/all-packages.nix | 6 +-- 4 files changed, 51 insertions(+), 30 deletions(-) delete mode 100644 pkgs/development/tools/continuous-integration/buildkite-agent/2.x.nix delete mode 100644 pkgs/development/tools/continuous-integration/buildkite-agent/3.x.nix create mode 100644 pkgs/development/tools/continuous-integration/buildkite-agent/default.nix diff --git a/pkgs/development/tools/continuous-integration/buildkite-agent/2.x.nix b/pkgs/development/tools/continuous-integration/buildkite-agent/2.x.nix deleted file mode 100644 index 6a73e2581822..000000000000 --- a/pkgs/development/tools/continuous-integration/buildkite-agent/2.x.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ callPackage, fetchFromGitHub, ... } @ args: - -callPackage ./generic.nix (args // rec { - src = fetchFromGitHub { - owner = "buildkite"; - repo = "agent"; - rev = "v${version}"; - sha256 = "07065hhhb418w5qlqnyiap45r59paysysbwz1l7dmaw3j4q8m8rg"; - }; - version = "2.6.10"; - hasBootstrapScript = true; -}) diff --git a/pkgs/development/tools/continuous-integration/buildkite-agent/3.x.nix b/pkgs/development/tools/continuous-integration/buildkite-agent/3.x.nix deleted file mode 100644 index f4dd4de93b2d..000000000000 --- a/pkgs/development/tools/continuous-integration/buildkite-agent/3.x.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ bash, callPackage, fetchFromGitHub, ... } @ args: - -callPackage ./generic.nix (args // rec { - src = fetchFromGitHub { - owner = "buildkite"; - repo = "agent"; - rev = "v${version}"; - sha256 = "0a7x919kxnpdn0pnhc5ilx1z6ninx8zgjvsd0jcg4qwh0qqp5ppr"; - }; - version = "3.17.0"; - hasBootstrapScript = false; - postPatch = '' - substituteInPlace bootstrap/shell/shell.go --replace /bin/bash ${bash}/bin/bash - ''; -}) diff --git a/pkgs/development/tools/continuous-integration/buildkite-agent/default.nix b/pkgs/development/tools/continuous-integration/buildkite-agent/default.nix new file mode 100644 index 000000000000..7e86c93b9bd6 --- /dev/null +++ b/pkgs/development/tools/continuous-integration/buildkite-agent/default.nix @@ -0,0 +1,48 @@ +{ fetchFromGitHub, stdenv, buildGoPackage, + makeWrapper, coreutils, git, openssh, bash, gnused, gnugrep }: +buildGoPackage rec { + name = "buildkite-agent-${version}"; + version = "3.17.0"; + + goPackagePath = "github.com/buildkite/agent"; + + src = fetchFromGitHub { + owner = "buildkite"; + repo = "agent"; + rev = "v${version}"; + sha256 = "0a7x919kxnpdn0pnhc5ilx1z6ninx8zgjvsd0jcg4qwh0qqp5ppr"; + }; + postPatch = '' + substituteInPlace bootstrap/shell/shell.go --replace /bin/bash ${bash}/bin/bash + ''; + + nativeBuildInputs = [ makeWrapper ]; + + # on Linux, the TMPDIR is /build which is the same prefix as this package + # remove once #35068 is merged + noAuditTmpdir = stdenv.isLinux; + + postInstall = '' + # Fix binary name + mv $bin/bin/{agent,buildkite-agent} + + # These are runtime dependencies + wrapProgram $bin/bin/buildkite-agent \ + --prefix PATH : '${stdenv.lib.makeBinPath [ openssh git coreutils gnused gnugrep ]}' + ''; + + meta = with stdenv.lib; { + description = "Build runner for buildkite.com"; + longDescription = '' + The buildkite-agent is a small, reliable, and cross-platform build runner + that makes it easy to run automated builds on your own infrastructure. + It’s main responsibilities are polling buildkite.com for work, running + build jobs, reporting back the status code and output log of the job, + and uploading the job's artifacts. + ''; + homepage = https://buildkite.com/docs/agent; + license = licenses.mit; + maintainers = with maintainers; [ pawelpacana zimbatm rvl ]; + platforms = platforms.unix; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index a748894887fe..55005036c262 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -9768,9 +9768,9 @@ in buck = callPackage ../development/tools/build-managers/buck { }; - buildkite-agent = buildkite-agent2; - buildkite-agent2 = callPackage ../development/tools/continuous-integration/buildkite-agent/2.x.nix { }; - buildkite-agent3 = callPackage ../development/tools/continuous-integration/buildkite-agent/3.x.nix { }; + buildkite-agent = buildkite-agent3; + buildkite-agent2 = throw "pkgs.buildkite-agent2 has been discontinued. Please use pkgs.buildkite-agent (v3.x)"; + buildkite-agent3 = callPackage ../development/tools/continuous-integration/buildkite-agent { }; libbpf = callPackage ../os-specific/linux/libbpf { }; From e1ad9d16857d53aab8762249af7bd566856ed81a Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Thu, 6 Sep 2018 14:02:55 +0200 Subject: [PATCH 5/7] nixos/buildkite-agent: add a shell attribute to specify the shell being used This gets passed to BUILDKITE_SHELL, which will specify the shell being used to executes script in. Defaults to `${pkgs.bash}/bin/bash -e -c`, matching how buildkite behaves on other distros. --- .../services/continuous-integration/buildkite-agent.nix | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/nixos/modules/services/continuous-integration/buildkite-agent.nix b/nixos/modules/services/continuous-integration/buildkite-agent.nix index 3da1e5e79e21..00b881ffe3c2 100644 --- a/nixos/modules/services/continuous-integration/buildkite-agent.nix +++ b/nixos/modules/services/continuous-integration/buildkite-agent.nix @@ -174,6 +174,14 @@ in instead. ''; }; + + shell = mkOption { + type = types.str; + default = "${pkgs.bash}/bin/bash -e -c"; + description = '' + Command that buildkite-agent 3 will execute when it spawns a shell. + ''; + }; }; }; @@ -216,6 +224,7 @@ in token="$(cat ${toString cfg.tokenPath})" name="${cfg.name}" ${metaData} + shell="${cfg.shell}" build-path="${cfg.dataDir}/builds" hooks-path="${cfg.hooksPath}" ${cfg.extraConfig} From 63c310c07eb04837130c26bc1149e9a19dd2b559 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Fri, 17 Jan 2020 17:12:35 +0100 Subject: [PATCH 6/7] nixos/buildkite-agent: change option meta-data into tags attrset This is called tags in the buildkite 3. --- .../continuous-integration/buildkite-agent.nix | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/nixos/modules/services/continuous-integration/buildkite-agent.nix b/nixos/modules/services/continuous-integration/buildkite-agent.nix index 00b881ffe3c2..3c9c92bf0527 100644 --- a/nixos/modules/services/continuous-integration/buildkite-agent.nix +++ b/nixos/modules/services/continuous-integration/buildkite-agent.nix @@ -74,13 +74,12 @@ in ''; }; - meta-data = mkOption { - type = types.str; - default = ""; - example = "queue=default,docker=true,ruby2=true"; + tags = mkOption { + type = types.attrsOf types.str; + default = {}; + example = { queue = "default"; docker = "true"; ruby2 ="true"; }; description = '' - Meta data for the agent. This is a comma-separated list of - key=value pairs. + Tags for the agent. ''; }; @@ -211,9 +210,7 @@ in ## don't end up in the Nix store. preStart = let sshDir = "${cfg.dataDir}/.ssh"; - metaData = if cfg.meta-data == "" - then "" - else "meta-data=${cfg.meta-data}"; + tagStr = lib.concatStringsSep "," (lib.mapAttrsToList (name: value: "${name}=${value}") cfg.tags); in '' mkdir -m 0700 -p "${sshDir}" @@ -223,8 +220,8 @@ in cat > "${cfg.dataDir}/buildkite-agent.cfg" < Date: Fri, 17 Jan 2020 18:32:23 +0100 Subject: [PATCH 7/7] nixos/buildkite: add release notes --- nixos/doc/manual/release-notes/rl-2003.xml | 38 ++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/nixos/doc/manual/release-notes/rl-2003.xml b/nixos/doc/manual/release-notes/rl-2003.xml index 51f91268eff0..b5385f151f59 100644 --- a/nixos/doc/manual/release-notes/rl-2003.xml +++ b/nixos/doc/manual/release-notes/rl-2003.xml @@ -401,6 +401,44 @@ users.users.me = the type to either path (submodule ...). + + + The Buildkite Agent + module and corresponding packages have been updated to 3.x. + While doing so, the following options have been changed: + + + + + services.buildkite-agent.meta-data has been renamed to + services.buildkite-agent.tags, + to match upstreams naming for 3.x. + Its type has also changed - it now accepts an attrset of strings. + + + + + Theservices.buildkite-agent.openssh.publicKeyPath option + has been removed, as it's not necessary to deploy public keys to clone private + repositories. + + + + + services.buildkite-agent.openssh.privateKeyPath + has been renamed to + buildkite-agent.privateSshKeyPath, + as the whole openssh now only contained that single option. + + + + + services.buildkite-agent.shell + has been introduced, allowing to specify a custom shell to be used. + + + +