Merge staging-next into staging

.github/CODEOWNERS

@@ -199,9 +199,9 @@
 /doc/languages-frameworks/php.section.md @NixOS/php
 /nixos/tests/php                         @NixOS/php
 /pkgs/build-support/build-pecl.nix       @NixOS/php
-/pkgs/development/interpreters/php       @NixOS/php
+/pkgs/development/interpreters/php       @NixOS/php @jtojnar
 /pkgs/development/php-packages           @NixOS/php
-/pkgs/top-level/php-packages.nix         @NixOS/php
+/pkgs/top-level/php-packages.nix         @NixOS/php @jtojnar
 
 # Podman, CRI-O modules and related
 /nixos/modules/virtualisation/containers.nix @NixOS/podman @zowoq

nixos/modules/module-list.nix

@@ -594,6 +594,7 @@
   ./services/monitoring/loki.nix
   ./services/monitoring/longview.nix
   ./services/monitoring/mackerel-agent.nix
+  ./services/monitoring/metricbeat.nix
   ./services/monitoring/monit.nix
   ./services/monitoring/munin.nix
   ./services/monitoring/nagios.nix

nixos/modules/services/monitoring/metricbeat.nix

@@ -0,0 +1,152 @@
+{ config, lib, pkgs, ... }:
+
+let
+  inherit (lib)
+    attrValues
+    literalExample
+    mkEnableOption
+    mkIf
+    mkOption
+    types
+    ;
+  cfg = config.services.metricbeat;
+
+  settingsFormat = pkgs.formats.yaml {};
+
+in
+{
+  options = {
+
+    services.metricbeat = {
+
+      enable = mkEnableOption "metricbeat";
+
+      package = mkOption {
+        type = types.package;
+        default = pkgs.metricbeat;
+        defaultText = literalExample "pkgs.metricbeat";
+        example = literalExample "pkgs.metricbeat7";
+        description = ''
+          The metricbeat package to use
+        '';
+      };
+
+      modules = mkOption {
+        description = ''
+          Metricbeat modules are responsible for reading metrics from the various sources.
+
+          This is like <literal>services.metricbeat.settings.metricbeat.modules</literal>,
+          but structured as an attribute set. This has the benefit that multiple
+          NixOS modules can contribute settings to a single metricbeat module.
+
+          A module can be specified multiple times by choosing a different <literal>&lt;name></literal>
+          for each, but setting <xref linkend="opt-services.metricbeat.modules._name_.module"/> to the same value.
+
+          See <link xlink:href="https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-modules.html"/>.
+        '';
+        default = {};
+        type = types.attrsOf (types.submodule ({ name, ... }: {
+          freeformType = settingsFormat.type;
+          options = {
+            module = mkOption {
+              type = types.str;
+              default = name;
+              defaultText = literalExample ''<name>'';
+              description = ''
+                The name of the module.
+
+                Look for the value after <literal>module:</literal> on the individual
+                module pages linked from <link xlink:href="https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-modules.html"/>.
+              '';
+            };
+          };
+        }));
+        example = {
+          system = {
+            metricsets = ["cpu" "load" "memory" "network" "process" "process_summary" "uptime" "socket_summary"];
+            enabled = true;
+            period = "10s";
+            processes = [".*"];
+            cpu.metrics = ["percentages" "normalized_percentages"];
+            core.metrics = ["percentages"];
+          };
+        };
+      };
+
+      settings = mkOption {
+        type = types.submodule {
+          freeformType = settingsFormat.type;
+          options = {
+
+            name = mkOption {
+              type = types.str;
+              default = "";
+              description = ''
+                Name of the beat. Defaults to the hostname.
+                See <link xlink:href="https://www.elastic.co/guide/en/beats/metricbeat/current/configuration-general-options.html#_name"/>.
+              '';
+            };
+
+            tags = mkOption {
+              type = types.listOf types.str;
+              default = [];
+              description = ''
+                Tags to place on the shipped metrics.
+                See <link xlink:href="https://www.elastic.co/guide/en/beats/metricbeat/current/configuration-general-options.html#_tags_2"/>.
+              '';
+            };
+
+            metricbeat.modules = mkOption {
+              type = types.listOf settingsFormat.type;
+              default = [];
+              internal = true;
+              description = ''
+                The metric collecting modules. Use <xref linkend="opt-services.metricbeat.modules"/> instead.
+
+                See <link xlink:href="https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-modules.html"/>.
+              '';
+            };
+          };
+        };
+        default = {};
+        description = ''
+          Configuration for metricbeat. See <link xlink:href="https://www.elastic.co/guide/en/beats/metricbeat/current/configuring-howto-metricbeat.html"/> for supported values.
+        '';
+      };
+
+    };
+  };
+
+  config = mkIf cfg.enable {
+
+    assertions = [
+      {
+        # empty modules would cause a failure at runtime
+        assertion = cfg.settings.metricbeat.modules != [];
+        message = "services.metricbeat: You must configure one or more modules.";
+      }
+    ];
+
+    services.metricbeat.settings.metricbeat.modules = attrValues cfg.modules;
+
+    systemd.services.metricbeat = {
+      description = "metricbeat metrics shipper";
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        ExecStart = ''
+          ${cfg.package}/bin/metricbeat \
+            -c ${settingsFormat.generate "metricbeat.yml" cfg.settings} \
+            --path.data $STATE_DIRECTORY \
+            --path.logs $LOGS_DIRECTORY \
+            ;
+        '';
+        Restart = "always";
+        DynamicUser = true;
+        ProtectSystem = "strict";
+        ProtectHome = "tmpfs";
+        StateDirectory = "metricbeat";
+        LogsDirectory = "metricbeat";
+      };
+    };
+  };
+}

nixos/tests/elk.nix

@@ -56,6 +56,24 @@ let
                 '');
               };
 
+              metricbeat = {
+                enable = true;
+                package = elk.metricbeat;
+                modules.system = {
+                  metricsets = ["cpu" "load" "memory" "network" "process" "process_summary" "uptime" "socket_summary"];
+                  enabled = true;
+                  period = "5s";
+                  processes = [".*"];
+                  cpu.metrics = ["percentages" "normalized_percentages"];
+                  core.metrics = ["percentages"];
+                };
+                settings = {
+                  output.elasticsearch = {
+                    hosts = ["127.0.0.1:9200"];
+                  };
+                };
+              };
+
               logstash = {
                 enable = true;
                 package = elk.logstash;
@@ -135,6 +153,16 @@ let
           )
 
 
+      def has_metricbeat():
+          dictionary = {"query": {"match": {"event.dataset": {"query": "system.cpu"}}}}
+          return (
+              "curl --silent --show-error '${esUrl}/_search' "
+              + "-H 'Content-Type: application/json' "
+              + "-d '{}' ".format(json.dumps(dictionary))
+              + "| jq '.hits.total > 0'"
+          )
+
+
       start_all()
 
       one.wait_for_unit("elasticsearch.service")
@@ -161,6 +189,12 @@ let
               "curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green"
           )
 
+      with subtest("Metricbeat is running"):
+          one.wait_for_unit("metricbeat.service")
+
+      with subtest("Metricbeat metrics arrive in elasticsearch"):
+          one.wait_until_succeeds(has_metricbeat() + " | tee /dev/console | grep 'true'")
+
       with subtest("Logstash messages arive in elasticsearch"):
           one.wait_until_succeeds(total_hits("flowers") + " | grep -v 0")
           one.wait_until_succeeds(total_hits("dragons") + " | grep 0")
@@ -190,12 +224,14 @@ in pkgs.lib.mapAttrs mkElkTest {
       logstash      = pkgs.logstash6;
       kibana        = pkgs.kibana6;
       journalbeat   = pkgs.journalbeat6;
+      metricbeat    = pkgs.metricbeat6;
     }
     else {
       elasticsearch = pkgs.elasticsearch6-oss;
       logstash      = pkgs.logstash6-oss;
       kibana        = pkgs.kibana6-oss;
       journalbeat   = pkgs.journalbeat6;
+      metricbeat    = pkgs.metricbeat6;
     };
   ELK-7 =
     if enableUnfree
@@ -204,11 +240,13 @@ in pkgs.lib.mapAttrs mkElkTest {
       logstash      = pkgs.logstash7;
       kibana        = pkgs.kibana7;
       journalbeat   = pkgs.journalbeat7;
+      metricbeat    = pkgs.metricbeat7;
     }
     else {
       elasticsearch = pkgs.elasticsearch7-oss;
       logstash      = pkgs.logstash7-oss;
       kibana        = pkgs.kibana7-oss;
       journalbeat   = pkgs.journalbeat7;
+      metricbeat    = pkgs.metricbeat7;
     };
 }

pkgs/development/interpreters/php/7.4.nix

@@ -1,9 +1,7 @@
-{ callPackage, lib, stdenv, nixosTests, ... }@_args:
+{ callPackage, lib, stdenv, ... }@_args:
 
 let
-  generic = (import ./generic.nix) _args;
-
-  base = callPackage generic (_args // {
+  base = callPackage ./generic.nix (_args // {
     version = "7.4.20";
     sha256 = "0d5ncz97y0271dsmz269wl4721vhq2fn6pmm9rxglc756p36pnha";
   });

pkgs/development/interpreters/php/8.0.nix

@@ -1,9 +1,7 @@
-{ callPackage, lib, stdenv, nixosTests, ... }@_args:
+{ callPackage, lib, stdenv, ... }@_args:
 
 let
-  generic = (import ./generic.nix) _args;
-
-  base = callPackage generic (_args // {
+  base = callPackage ./generic.nix (_args // {
     version = "8.0.7";
     sha256 = "0yazcc9x66xg1gmi3rpgk891g6s3mm7aywcadqfqnx1mdz4z5ckj";
   });

pkgs/development/interpreters/php/generic.nix

@@ -1,8 +1,6 @@
 # We have tests for PCRE and PHP-FPM in nixos/tests/php/ or
 # both in the same attribute named nixosTests.php
 
-{ callPackage, lib, stdenv, nixosTests, ... }:
-
 let
   generic =
     { callPackage, lib, stdenv, nixosTests, fetchurl, makeWrapper
@@ -33,6 +31,7 @@ let
     , valgrindSupport ? !stdenv.isDarwin
     , ztsSupport ? apxs2Support
     }@args:
+
     let
       # buildEnv wraps php to provide additional extensions and
       # configuration. Its usage is documented in

pkgs/development/libraries/libimagequant/default.nix

@@ -15,7 +15,7 @@ stdenv.mkDerivation rec {
     patchShebangs ./configure
   '';
 
-  configureFlags = lib.optionals stdenv.isAarch64 [ "--disable-sse" ];
+  configureFlags = lib.optionals (!stdenv.isi686 && !stdenv.isx86_64) [ "--disable-sse" ];
 
   meta = with lib; {
     homepage = "https://pngquant.org/lib/";

pkgs/development/php-packages/composer/default.nix

@@ -1,14 +1,14 @@
 { mkDerivation, fetchurl, makeWrapper, unzip, lib, php }:
 let
   pname = "composer";
-  version = "2.1.1";
+  version = "2.1.2";
 in
 mkDerivation {
   inherit pname version;
 
   src = fetchurl {
     url = "https://getcomposer.org/download/${version}/composer.phar";
-    sha256 = "1ki106973q74inwgd4hjmml905rqg82808qq4wiysrkr7mzmfnj4";
+    sha256 = "0gd4hxkxdds3nxpbcd38chrkijha31p6nygdq3f73mbb984h3v1d";
   };
 
   dontUnpack = true;

pkgs/development/tools/buildah/default.nix

@@ -14,13 +14,13 @@
 
 buildGoModule rec {
   pname = "buildah";
-  version = "1.21.0";
+  version = "1.21.1";
 
   src = fetchFromGitHub {
     owner = "containers";
     repo = "buildah";
     rev = "v${version}";
-    sha256 = "sha256-uNb5HCEft1vXASli+2zdKWzFzsAlI9/ILBWa7OQZBwE=";
+    sha256 = "sha256-Wes52lTcv3Jb6gJeUS6fmf4Nee3qEcc3SibaTFvQ8sQ=";
   };
 
   outputs = [ "out" "man" ];

pkgs/misc/logging/beats/6.x.nix

@@ -1,4 +1,4 @@
-{ lib, fetchFromGitHub, elk6Version, buildGoPackage, libpcap, systemd }:
+{ lib, fetchFromGitHub, elk6Version, buildGoPackage, libpcap, nixosTests, systemd }:
 
 let beat = package : extraArgs : buildGoPackage (rec {
       name = "${package}-${version}";
@@ -22,10 +22,17 @@ let beat = package : extraArgs : buildGoPackage (rec {
         platforms = platforms.linux;
       };
     } // extraArgs);
-in {
+in rec {
   filebeat6   = beat "filebeat"   {meta.description = "Lightweight shipper for logfiles";};
   heartbeat6  = beat "heartbeat"  {meta.description = "Lightweight shipper for uptime monitoring";};
-  metricbeat6 = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";};
+  metricbeat6 = beat "metricbeat" {
+    meta.description = "Lightweight shipper for metrics";
+    passthru.tests =
+      assert metricbeat6.drvPath == nixosTests.elk.ELK-6.elkPackages.metricbeat.drvPath;
+      {
+        elk = nixosTests.elk.ELK-6;
+      };
+  };
   packetbeat6 = beat "packetbeat" {
     buildInputs = [ libpcap ];
     meta.broken = true;

pkgs/misc/logging/beats/7.x.nix

@@ -1,4 +1,4 @@
-{ lib, fetchFromGitHub, elk7Version, buildGoPackage, libpcap, systemd }:
+{ lib, fetchFromGitHub, elk7Version, buildGoPackage, libpcap, nixosTests, systemd }:
 
 let beat = package : extraArgs : buildGoPackage (rec {
       name = "${package}-${version}";
@@ -22,10 +22,17 @@ let beat = package : extraArgs : buildGoPackage (rec {
         platforms = platforms.linux;
       };
     } // extraArgs);
-in {
+in rec {
   filebeat7   = beat "filebeat"   {meta.description = "Lightweight shipper for logfiles";};
   heartbeat7  = beat "heartbeat"  {meta.description = "Lightweight shipper for uptime monitoring";};
-  metricbeat7 = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";};
+  metricbeat7 = beat "metricbeat" {
+    meta.description = "Lightweight shipper for metrics";
+    passthru.tests =
+      assert metricbeat7.drvPath == nixosTests.elk.ELK-7.elkPackages.metricbeat.drvPath;
+      {
+        elk = nixosTests.elk.ELK-7;
+      };
+  };
   packetbeat7 = beat "packetbeat" {
     buildInputs = [ libpcap ];
     meta.description = "Network packet analyzer that ships data to Elasticsearch";

pkgs/servers/ldap/389/default.nix

@@ -65,5 +65,8 @@ stdenv.mkDerivation rec {
     description = "Enterprise-class Open Source LDAP server for Linux";
     license = licenses.gpl3Plus;
     platforms = platforms.linux;
+    knownVulnerabilities = [
+      "CVE-2021-3514" # https://nvd.nist.gov/vuln/detail/CVE-2021-3514
+    ];
   };
 }

pkgs/servers/misc/gobgpd/default.nix

@@ -2,13 +2,13 @@
 
 buildGoModule rec {
   pname = "gobgpd";
-  version = "2.27.0";
+  version = "2.28.0";
 
   src = fetchFromGitHub {
     owner = "osrg";
     repo = "gobgp";
     rev = "v${version}";
-    sha256 = "sha256-Ofg+z8wUttqM1THatPFi0cuyLSEryhTmg3JC1o+16eA=";
+    sha256 = "sha256-AlAfs1wi3hS7cqAtGqfUgv5CIonu7TmXWgxL1zSBh54=";
   };
 
   vendorSha256 = "sha256-PWm7XnO6LPaU8g8ymmqRkQv2KSX9kLv9RVaa000mrTY=";

pkgs/tools/filesystems/gocryptfs/default.nix

@@ -1,28 +1,42 @@
 { lib
+, stdenv
 , buildGoModule
 , fetchFromGitHub
 , openssl
 , pandoc
 , pkg-config
+, libfido2
 }:
 
+let
+  # pandoc is currently broken on aarch64-darwin
+  # because of missing ghc
+  brokenPandoc = stdenv.isDarwin && stdenv.isAarch64;
+in
+
 buildGoModule rec {
   pname = "gocryptfs";
-  version = "1.8.0";
+  version = "2.0";
 
   src = fetchFromGitHub {
     owner = "rfjakob";
     repo = pname;
     rev = "v${version}";
-    sha256 = "1acalwrr5xqhpqca3gypj0s68w6vpckxmg5z5gfgh8wx6nqx4aw9";
+    sha256 = "1wpdzi1qfpab76v0ki74qkk82m3ykr4iqb8r6a8k11l4fn42fjk0";
   };
 
-  runVend = true;
-  vendorSha256 = "0z3y51sgr1rmr23jpc5h5d5lw14p3qzv48rc7zj7qa4rd5cfhsgi";
+  vendorSha256 = "10az8n7z4rhsk1af2x6v3pmxg4zp7c9cal35ily8bdzzcb9cpgs0";
+
+  nativeBuildInputs = [
+    pkg-config
+  ] ++ lib.optionals (!brokenPandoc) [
+    pandoc
+  ];
 
-  nativeBuildInputs = [ pandoc pkg-config ];
   buildInputs = [ openssl ];
 
+  propagatedBuildInputs = [ libfido2 ];
+
   buildFlagsArray = ''
     -ldflags=
       -X main.GitVersion=${version}
@@ -32,9 +46,10 @@ buildGoModule rec {
 
   subPackages = [ "." "gocryptfs-xray" "contrib/statfs" ];
 
-  postBuild = ''
+  postBuild = lib.optionalString (!brokenPandoc) ''
     pushd Documentation/
     mkdir -p $out/share/man/man1
+    # taken from Documentation/MANPAGE-render.bash
     pandoc MANPAGE.md -s -t man -o $out/share/man/man1/gocryptfs.1
     pandoc MANPAGE-XRAY.md -s -t man -o $out/share/man/man1/gocryptfs-xray.1
     pandoc MANPAGE-STATFS.md -s -t man -o $out/share/man/man1/statfs.1

pkgs/tools/misc/nix-direnv/default.nix

@@ -1,5 +1,15 @@
-{ lib, stdenv, fetchFromGitHub, gnugrep, nixUnstable }:
+{ lib
+, stdenv
+, fetchFromGitHub
+, gnugrep
+, nixStable
+, nixUnstable
+, enableFlakes ? false
+}:
 
+let
+  nix = if enableFlakes then nixUnstable else nixStable;
+in
 stdenv.mkDerivation rec {
   pname = "nix-direnv";
   version = "1.2.6";
@@ -14,7 +24,7 @@ stdenv.mkDerivation rec {
   # Substitute instead of wrapping because the resulting file is
   # getting sourced, not executed:
   postPatch = ''
-    sed -i "1a NIX_BIN_PREFIX=${nixUnstable}/bin/" direnvrc
+    sed -i "1a NIX_BIN_PREFIX=${nix}/bin/" direnvrc
     substituteInPlace direnvrc --replace "grep" "${gnugrep}/bin/grep"
   '';
 

pkgs/tools/networking/gobgp/default.nix

@@ -2,13 +2,13 @@
 
 buildGoModule rec {
   pname = "gobgp";
-  version = "2.27.0";
+  version = "2.28.0";
 
   src = fetchFromGitHub {
     owner = "osrg";
     repo = "gobgp";
     rev = "v${version}";
-    sha256 = "sha256-Ofg+z8wUttqM1THatPFi0cuyLSEryhTmg3JC1o+16eA=";
+    sha256 = "sha256-AlAfs1wi3hS7cqAtGqfUgv5CIonu7TmXWgxL1zSBh54=";
   };
 
   vendorSha256 = "sha256-PWm7XnO6LPaU8g8ymmqRkQv2KSX9kLv9RVaa000mrTY=";

pkgs/tools/system/gdu/default.nix

@@ -7,16 +7,16 @@
 
 buildGoModule rec {
   pname = "gdu";
-  version = "5.0.1";
+  version = "5.1.0";
 
   src = fetchFromGitHub {
     owner = "dundee";
     repo = pname;
     rev = "v${version}";
-    sha256 = "sha256-78eQinKR4w7K8MFd4uyj5IPkUs0Mz5XeO7JUG/1cKLw=";
+    sha256 = "sha256-OellGxW/2I/dKBxWgEv1Ta9OJ/2HUfDIzICQwvmjTCM=";
   };
 
-  vendorSha256 = "sha256-9upXhTvQJ9oFfomgqja3SiifiZpl8RUQ85HwL9bDPlQ=";
+  vendorSha256 = "sha256-9W1K01PJ+tRLSJ0L7NGHXT5w5oHmlBkT8kwnOLOzSCc=";
 
   nativeBuildInputs = [ installShellFiles ];