Merge pull request #77950 from flokli/buildkite-3

nixos/buildkite-agent: move to v3

nixos/doc/manual/release-notes/rl-2003.xml

@@ -407,6 +407,44 @@ users.users.me =
      the type to <literal>either path (submodule ...)</literal>.
     </para>
    </listitem>
+   <listitem>
+    <para>
+      The <link linkend="opt-services.buildkite-agent.enable">Buildkite Agent</link>
+      module and corresponding packages have been updated to 3.x.
+      While doing so, the following options have been changed:
+    </para>
+    <itemizedlist>
+      <listitem>
+       <para>
+         <literal>services.buildkite-agent.meta-data</literal> has been renamed to
+         <link linkend="opt-services.buildkite-agent.tags">services.buildkite-agent.tags</link>,
+         to match upstreams naming for 3.x.
+         Its type has also changed - it now accepts an attrset of strings.
+       </para>
+      </listitem>
+      <listitem>
+       <para>
+         The<literal>services.buildkite-agent.openssh.publicKeyPath</literal> option
+         has been removed, as it's not necessary to deploy public keys to clone private
+         repositories.
+       </para>
+      </listitem>
+      <listitem>
+       <para>
+         <literal>services.buildkite-agent.openssh.privateKeyPath</literal>
+         has been renamed to
+         <link linkend="opt-services.buildkite-agent.privateSshKeyPath">buildkite-agent.privateSshKeyPath</link>,
+         as the whole <literal>openssh</literal> now only contained that single option.
+       </para>
+      </listitem>
+      <listitem>
+       <para>
+         <link linkend="opt-services.buildkite-agent.shell">services.buildkite-agent.shell</link>
+         has been introduced, allowing to specify a custom shell to be used.
+       </para>
+      </listitem>
+    </itemizedlist>
+   </listitem>
   </itemizedlist>
  </section>
 

nixos/modules/services/continuous-integration/buildkite-agent.nix

@@ -74,13 +74,12 @@ in
         '';
       };
 
-      meta-data = mkOption {
-        type = types.str;
-        default = "";
-        example = "queue=default,docker=true,ruby2=true";
+      tags = mkOption {
+        type = types.attrsOf types.str;
+        default = {};
+        example = { queue = "default"; docker = "true"; ruby2 ="true"; };
         description = ''
-          Meta data for the agent. This is a comma-separated list of
-          <code>key=value</code> pairs.
+          Tags for the agent.
         '';
       };
 
@@ -93,26 +92,19 @@ in
         '';
       };
 
-      openssh =
-        { privateKeyPath = mkOption {
-            type = types.path;
-            description = ''
-              Private agent key.
+      privateSshKeyPath = mkOption {
+        type = types.path;
+        ## maximum care is taken so that secrets (ssh keys and the CI token)
+        ## don't end up in the Nix store.
+        apply = final: if final == null then null else toString final;
 
-              A run-time path to the key file, which is supposed to be provisioned
-              outside of Nix store.
-            '';
-          };
-          publicKeyPath = mkOption {
-            type = types.path;
-            description = ''
-              Public agent key.
+        description = ''
+          OpenSSH private key
 
-              A run-time path to the key file, which is supposed to be provisioned
-              outside of Nix store.
-            '';
-          };
-        };
+          A run-time path to the key file, which is supposed to be provisioned
+          outside of Nix store.
+        '';
+      };
 
       hooks = mkHookOptions [
         { name = "checkout";
@@ -181,6 +173,14 @@ in
           instead.
         '';
       };
+
+      shell = mkOption {
+        type = types.str;
+        default = "${pkgs.bash}/bin/bash -e -c";
+        description = ''
+          Command that buildkite-agent 3 will execute when it spawns a shell.
+        '';
+      };
     };
   };
 
@@ -210,20 +210,18 @@ in
         ##     don't end up in the Nix store.
         preStart = let
           sshDir = "${cfg.dataDir}/.ssh";
-          metaData = if cfg.meta-data == ""
-            then ""
-            else "meta-data=${cfg.meta-data}";
+          tagStr = lib.concatStringsSep "," (lib.mapAttrsToList (name: value: "${name}=${value}") cfg.tags);
         in
           ''
             mkdir -m 0700 -p "${sshDir}"
             cp -f "${toString cfg.openssh.privateKeyPath}" "${sshDir}/id_rsa"
-            cp -f "${toString cfg.openssh.publicKeyPath}"  "${sshDir}/id_rsa.pub"
             chmod 600 "${sshDir}"/id_rsa*
 
             cat > "${cfg.dataDir}/buildkite-agent.cfg" <<EOF
             token="$(cat ${toString cfg.tokenPath})"
             name="${cfg.name}"
-            ${metaData}
+            shell="${cfg.shell}"
+            tags="${tagStr}"
             build-path="${cfg.dataDir}/builds"
             hooks-path="${cfg.hooksPath}"
             ${cfg.extraConfig}
@@ -231,11 +229,14 @@ in
           '';
 
         serviceConfig =
-          { ExecStart = "${cfg.buildkite-agent}/bin/buildkite-agent start --config /var/lib/buildkite-agent/buildkite-agent.cfg";
+          { ExecStart = "${cfg.package}/bin/buildkite-agent start --config /var/lib/buildkite-agent/buildkite-agent.cfg";
             User = "buildkite-agent";
             RestartSec = 5;
             Restart = "on-failure";
             TimeoutSec = 10;
+            # set a long timeout to give buildkite-agent a chance to finish current builds
+            TimeoutStopSec = "2 min";
+            KillMode = "mixed";
           };
       };
 
@@ -249,8 +250,11 @@ in
     ];
   };
   imports = [
-    (mkRenamedOptionModule [ "services" "buildkite-agent" "token" ]                [ "services" "buildkite-agent" "tokenPath" ])
-    (mkRenamedOptionModule [ "services" "buildkite-agent" "openssh" "privateKey" ] [ "services" "buildkite-agent" "openssh" "privateKeyPath" ])
-    (mkRenamedOptionModule [ "services" "buildkite-agent" "openssh" "publicKey" ]  [ "services" "buildkite-agent" "openssh" "publicKeyPath" ])
+    (mkRenamedOptionModule [ "services" "buildkite-agent" "token" ]                    [ "services" "buildkite-agent" "tokenPath" ])
+    (mkRenamedOptionModule [ "services" "buildkite-agent" "openssh" "privateKey" ]     [ "services" "buildkite-agent" "privateSshKeyPath" ])
+    (mkRenamedOptionModule [ "services" "buildkite-agent" "openssh" "privateKeyPath" ] [ "services" "buildkite-agent" "privateSshKeyPath" ])
+    (mkRemovedOptionModule [ "services" "buildkite-agent" "openssh" "publicKey" ]      "SSH public keys aren't necessary to clone private repos.")
+    (mkRemovedOptionModule [ "services" "buildkite-agent" "openssh" "publicKeyPath" ]  "SSH public keys aren't necessary to clone private repos.")
+    (mkRenamedOptionModule [ "services" "buildkite-agent" "meta-data"]                 [ "services" "buildkite-agent" "tags" ])
   ];
 }

pkgs/development/tools/continuous-integration/buildkite-agent/2.x.nix

@@ -1,12 +0,0 @@
-{ callPackage, fetchFromGitHub, ... } @ args:
-
-callPackage ./generic.nix (args // rec {
-  src = fetchFromGitHub {
-    owner = "buildkite";
-    repo = "agent";
-    rev = "v${version}";
-    sha256 = "07065hhhb418w5qlqnyiap45r59paysysbwz1l7dmaw3j4q8m8rg";
-  };
-  version = "2.6.10";
-  hasBootstrapScript = true;
-})

pkgs/development/tools/continuous-integration/buildkite-agent/3.x.nix

@@ -1,15 +0,0 @@
-{ bash, callPackage, fetchFromGitHub, ... } @ args:
-
-callPackage ./generic.nix (args // rec {
-  src = fetchFromGitHub {
-    owner = "buildkite";
-    repo = "agent";
-    rev = "v${version}";
-    sha256 = "0a7x919kxnpdn0pnhc5ilx1z6ninx8zgjvsd0jcg4qwh0qqp5ppr";
-  };
-  version = "3.17.0";
-  hasBootstrapScript = false;
-  postPatch = ''
-    substituteInPlace bootstrap/shell/shell.go --replace /bin/bash ${bash}/bin/bash
-  '';
-})

pkgs/development/tools/continuous-integration/buildkite-agent/default.nix

@@ -0,0 +1,48 @@
+{ fetchFromGitHub, stdenv, buildGoPackage,
+  makeWrapper, coreutils, git, openssh, bash, gnused, gnugrep }:
+buildGoPackage rec {
+  name = "buildkite-agent-${version}";
+  version = "3.17.0";
+
+  goPackagePath = "github.com/buildkite/agent";
+
+  src = fetchFromGitHub {
+    owner = "buildkite";
+    repo = "agent";
+    rev = "v${version}";
+    sha256 = "0a7x919kxnpdn0pnhc5ilx1z6ninx8zgjvsd0jcg4qwh0qqp5ppr";
+  };
+  postPatch = ''
+    substituteInPlace bootstrap/shell/shell.go --replace /bin/bash ${bash}/bin/bash
+  '';
+
+  nativeBuildInputs = [ makeWrapper ];
+
+  # on Linux, the TMPDIR is /build which is the same prefix as this package
+  # remove once #35068 is merged
+  noAuditTmpdir = stdenv.isLinux;
+
+  postInstall = ''
+    # Fix binary name
+    mv $bin/bin/{agent,buildkite-agent}
+
+    # These are runtime dependencies
+    wrapProgram $bin/bin/buildkite-agent \
+      --prefix PATH : '${stdenv.lib.makeBinPath [ openssh git coreutils gnused gnugrep ]}'
+  '';
+
+  meta = with stdenv.lib; {
+    description = "Build runner for buildkite.com";
+    longDescription = ''
+      The buildkite-agent is a small, reliable, and cross-platform build runner
+      that makes it easy to run automated builds on your own infrastructure.
+      It’s main responsibilities are polling buildkite.com for work, running
+      build jobs, reporting back the status code and output log of the job,
+      and uploading the job's artifacts.
+    '';
+    homepage = https://buildkite.com/docs/agent;
+    license = licenses.mit;
+    maintainers = with maintainers; [ pawelpacana zimbatm rvl ];
+    platforms = platforms.unix;
+  };
+}

pkgs/top-level/all-packages.nix

@@ -9780,9 +9780,9 @@ in
 
   buck = callPackage ../development/tools/build-managers/buck { };
 
-  buildkite-agent = buildkite-agent2;
-  buildkite-agent2 = callPackage ../development/tools/continuous-integration/buildkite-agent/2.x.nix { };
-  buildkite-agent3 = callPackage ../development/tools/continuous-integration/buildkite-agent/3.x.nix { };
+  buildkite-agent = buildkite-agent3;
+  buildkite-agent2 = throw "pkgs.buildkite-agent2 has been discontinued. Please use pkgs.buildkite-agent (v3.x)";
+  buildkite-agent3 = callPackage ../development/tools/continuous-integration/buildkite-agent { };
 
   libbpf = callPackage ../os-specific/linux/libbpf { };