alioth/nixpkgs
3
0
Fork 0
forked from mirrors/nixpkgs
Code Releases Activity

elk7: 7.11.1 -> 7.16.1, 6.8.3 -> 6.8.21 + add filebeat module and tests (#150879)

Browse source 
* elk7: 7.11.1 -> 7.16.1

* nixosTests.elk: Improve reliability and compatibility with ELK 7.x

- Use comparisons in jq instead of grepping
- Match for `.hits.total.value` if version >= 7, otherwise it always
  passes
- Make curl fail if requests fails

* nixos/filebeat: Add initial module and test

Filebeat is an open source file harvester, mostly used to fetch logs
files and feed them into logstash.

This module can be used instead of journalbeat if used with
`filebeat7` and configured with the `journald` input.

* python3Packages.parsedmarc.tests: Fix breakage

- Don't use the deprecated elasticsearch7-oss package
- Improve jq query robustness and add tracing

* rl-2205: Note the addition of the filebeat service

* elk6: 6.8.3 -> 6.8.21

The latest version includes a fix for CVE-2021-44228.

* nixos/journalbeat: Add a loose dependency on elasticsearch

Avoid unnecssary back-off when elasticsearch is running on the same
host.
This commit is contained in:
Kim Lindberger 2021-12-16 16:20:52 +01:00 committed by GitHub
parent 1a3873c93d
commit ebaa226853
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 414 additions and 94 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
nixos/doc/manual/from_md/release-notes/rl-2205.section.xml
View file

@ -19,7 +19,7 @@
</section>
<section xml:id="sec-release-22.05-new-services">
<title>New Services</title>
<itemizedlist spacing="compact">
<itemizedlist>
<listitem>
<para>
<link xlink:href="https://github.com/intel/linux-sgx#install-the-intelr-sgx-psw">aesmd</link>,
@ -28,6 +28,14 @@
<link linkend="opt-services.aesmd.enable">services.aesmd</link>.
</para>
</listitem>
<listitem>
<para>
<link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-overview.html">filebeat</link>,
a lightweight shipper for forwarding and centralizing log
data. Available as
<link linkend="opt-services.filebeat.enable">services.filebeat</link>.
</para>
</listitem>
</itemizedlist>
</section>
<section xml:id="sec-release-22.05-incompatibilities">

2
nixos/doc/manual/release-notes/rl-2205.section.md
View file

@ -10,6 +10,8 @@ In addition to numerous new and upgraded packages, this release has the followin
- [aesmd](https://github.com/intel/linux-sgx#install-the-intelr-sgx-psw), the Intel SGX Architectural Enclave Service Manager. Available as [services.aesmd](#opt-services.aesmd.enable).
- [filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-overview.html), a lightweight shipper for forwarding and centralizing log data. Available as [services.filebeat](#opt-services.filebeat.enable).
## Backward Incompatibilities {#sec-release-22.05-incompatibilities}
- `pkgs.ghc` now refers to `pkgs.targetPackages.haskellPackages.ghc`.

1
nixos/modules/module-list.nix
View file

@ -448,6 +448,7 @@
./services/hardware/xow.nix
./services/logging/SystemdJournal2Gelf.nix
./services/logging/awstats.nix
./services/logging/filebeat.nix
./services/logging/fluentd.nix
./services/logging/graylog.nix
./services/logging/heartbeat.nix

253
nixos/modules/services/logging/filebeat.nix Normal file
View file

@ -0,0 +1,253 @@
{ config, lib, utils, pkgs, ... }:
let
inherit (lib)
attrValues
literalExpression
mkEnableOption
mkIf
mkOption
types;
cfg = config.services.filebeat;
json = pkgs.formats.json {};
in
{
options = {
services.filebeat = {
enable = mkEnableOption "filebeat";
package = mkOption {
type = types.package;
default = pkgs.filebeat;
defaultText = literalExpression "pkgs.filebeat";
example = literalExpression "pkgs.filebeat7";
description = ''
The filebeat package to use.
'';
};
inputs = mkOption {
description = ''
Inputs specify how Filebeat locates and processes input data.
This is like <literal>services.filebeat.settings.filebeat.inputs</literal>,
but structured as an attribute set. This has the benefit
that multiple NixOS modules can contribute settings to a
single filebeat input.
An input type can be specified multiple times by choosing a
different <literal>&lt;name></literal> for each, but setting
<xref linkend="opt-services.filebeat.inputs._name_.type"/>
to the same value.
See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>.
'';
default = {};
type = types.attrsOf (types.submodule ({ name, ... }: {
freeformType = json.type;
options = {
type = mkOption {
type = types.str;
default = name;
description = ''
The input type.
Look for the value after <literal>type:</literal> on
the individual input pages linked from
<link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>.
'';
};
};
}));
example = literalExpression ''
{
journald.id = "everything"; # Only for filebeat7
log = {
enabled = true;
paths = [
"/var/log/*.log"
];
};
};
'';
};
modules = mkOption {
description = ''
Filebeat modules provide a quick way to get started
processing common log formats. They contain default
configurations, Elasticsearch ingest pipeline definitions,
and Kibana dashboards to help you implement and deploy a log
monitoring solution.
This is like <literal>services.filebeat.settings.filebeat.modules</literal>,
but structured as an attribute set. This has the benefit
that multiple NixOS modules can contribute settings to a
single filebeat module.
A module can be specified multiple times by choosing a
different <literal>&lt;name></literal> for each, but setting
<xref linkend="opt-services.filebeat.modules._name_.module"/>
to the same value.
See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>.
'';
default = {};
type = types.attrsOf (types.submodule ({ name, ... }: {
freeformType = json.type;
options = {
module = mkOption {
type = types.str;
default = name;
description = ''
The name of the module.
Look for the value after <literal>module:</literal> on
the individual input pages linked from
<link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>.
'';
};
};
}));
example = literalExpression ''
{
nginx = {
access = {
enabled = true;
var.paths = [ "/path/to/log/nginx/access.log*" ];
};
error = {
enabled = true;
var.paths = [ "/path/to/log/nginx/error.log*" ];
};
};
};
'';
};
settings = mkOption {
type = types.submodule {
freeformType = json.type;
options = {
output.elasticsearch.hosts = mkOption {
type = with types; listOf str;
default = [ "127.0.0.1:9200" ];
example = [ "myEShost:9200" ];
description = ''
The list of Elasticsearch nodes to connect to.
The events are distributed to these nodes in round
robin order. If one node becomes unreachable, the
event is automatically sent to another node. Each
Elasticsearch node can be defined as a URL or
IP:PORT. For example:
<literal>http://192.15.3.2</literal>,
<literal>https://es.found.io:9230</literal> or
<literal>192.24.3.2:9300</literal>. If no port is
specified, <literal>9200</literal> is used.
'';
};
filebeat = {
inputs = mkOption {
type = types.listOf json.type;
default = [];
internal = true;
description = ''
Inputs specify how Filebeat locates and processes
input data. Use <xref
linkend="opt-services.filebeat.inputs"/> instead.
See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>.
'';
};
modules = mkOption {
type = types.listOf json.type;
default = [];
internal = true;
description = ''
Filebeat modules provide a quick way to get started
processing common log formats. They contain default
configurations, Elasticsearch ingest pipeline
definitions, and Kibana dashboards to help you
implement and deploy a log monitoring solution.
Use <xref linkend="opt-services.filebeat.modules"/> instead.
See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>.
'';
};
};
};
};
default = {};
example = literalExpression ''
{
settings = {
output.elasticsearch = {
hosts = [ "myEShost:9200" ];
username = "filebeat_internal";
password = { _secret = "/var/keys/elasticsearch_password"; };
};
logging.level = "info";
};
};
'';
description = ''
Configuration for filebeat. See
<link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-reference-yml.html"/>
for supported values.
Options containing secret data should be set to an attribute
set containing the attribute <literal>_secret</literal> - a
string pointing to a file containing the value the option
should be set to. See the example to get a better picture of
this: in the resulting
<filename>filebeat.yml</filename> file, the
<literal>output.elasticsearch.password</literal>
key will be set to the contents of the
<filename>/var/keys/elasticsearch_password</filename> file.
'';
};
};
};
config = mkIf cfg.enable {
services.filebeat.settings.filebeat.inputs = attrValues cfg.inputs;
services.filebeat.settings.filebeat.modules = attrValues cfg.modules;
systemd.services.filebeat = {
description = "Filebeat log shipper";
wantedBy = [ "multi-user.target" ];
wants = [ "elasticsearch.service" ];
after = [ "elasticsearch.service" ];
serviceConfig = {
ExecStartPre = pkgs.writeShellScript "filebeat-exec-pre" ''
set -euo pipefail
umask u=rwx,g=,o=
${utils.genJqSecretsReplacementSnippet
cfg.settings
"/var/lib/filebeat/filebeat.yml"
}
'';
ExecStart = ''
${cfg.package}/bin/filebeat -e \
-c "/var/lib/filebeat/filebeat.yml" \
--path.data "/var/lib/filebeat"
'';
Restart = "always";
StateDirectory = "filebeat";
};
};
};
}

3
nixos/modules/services/logging/journalbeat.nix
View file

@ -28,7 +28,6 @@ in
type = types.package;
default = pkgs.journalbeat;
defaultText = literalExpression "pkgs.journalbeat";
example = literalExpression "pkgs.journalbeat7";
description = ''
The journalbeat package to use
'';
@ -89,6 +88,8 @@ in
systemd.services.journalbeat = {
description = "Journalbeat log shipper";
wantedBy = [ "multi-user.target" ];
wants = [ "elasticsearch.service" ];
after = [ "elasticsearch.service" ];
preStart = ''
mkdir -p ${cfg.stateDir}/data
mkdir -p ${cfg.stateDir}/logs

90
nixos/tests/elk.nix
View file

@ -40,9 +40,8 @@ let
services = {
journalbeat = let lt6 = builtins.compareVersions
elk.journalbeat.version "6" < 0; in {
enable = true;
journalbeat = {
enable = elk ? journalbeat;
package = elk.journalbeat;
extraConfig = pkgs.lib.mkOptionDefault (''
logging:
@ -51,14 +50,29 @@ let
metrics.enabled: false
output.elasticsearch:
hosts: [ "127.0.0.1:9200" ]
${pkgs.lib.optionalString lt6 "template.enabled: false"}
'' + pkgs.lib.optionalString (!lt6) ''
journalbeat.inputs:
- paths: []
seek: cursor
'');
};
filebeat = {
enable = elk ? filebeat;
package = elk.filebeat;
inputs.journald.id = "everything";
inputs.log = {
enabled = true;
paths = [
"/var/lib/filebeat/test"
];
};
settings = {
logging.level = "info";
};
};
metricbeat = {
enable = true;
package = elk.metricbeat;
@ -142,27 +156,43 @@ let
};
passthru.elkPackages = elk;
testScript = ''
testScript =
let
valueObject = lib.optionalString (lib.versionAtLeast elk.elasticsearch.version "7") ".value";
in ''
import json
def total_hits(message):
def expect_hits(message):
dictionary = {"query": {"match": {"message": message}}}
return (
"curl --silent --show-error '${esUrl}/_search' "
"curl --silent --show-error --fail-with-body '${esUrl}/_search' "
+ "-H 'Content-Type: application/json' "
+ "-d '{}' ".format(json.dumps(dictionary))
+ "| jq .hits.total"
+ " | tee /dev/console"
+ " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
)
def expect_no_hits(message):
dictionary = {"query": {"match": {"message": message}}}
return (
"curl --silent --show-error --fail-with-body '${esUrl}/_search' "
+ "-H 'Content-Type: application/json' "
+ "-d '{}' ".format(json.dumps(dictionary))
+ " | tee /dev/console"
+ " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} == 0 end'"
)
def has_metricbeat():
dictionary = {"query": {"match": {"event.dataset": {"query": "system.cpu"}}}}
return (
"curl --silent --show-error '${esUrl}/_search' "
"curl --silent --show-error --fail-with-body '${esUrl}/_search' "
+ "-H 'Content-Type: application/json' "
+ "-d '{}' ".format(json.dumps(dictionary))
+ "| jq '.hits.total > 0'"
+ " | tee /dev/console"
+ " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
)
@ -178,7 +208,8 @@ let
# TODO: extend this test with multiple elasticsearch nodes
# and see if the status turns "green".
one.wait_until_succeeds(
"curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red"
"curl --silent --show-error --fail-with-body '${esUrl}/_cluster/health'"
+ " | jq -es 'if . == [] then null else .[] | .status != \"red\" end'"
)
with subtest("Perform some simple logstash tests"):
@ -189,33 +220,50 @@ let
with subtest("Kibana is healthy"):
one.wait_for_unit("kibana.service")
one.wait_until_succeeds(
"curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green"
"curl --silent --show-error --fail-with-body 'http://localhost:5601/api/status'"
+ " | jq -es 'if . == [] then null else .[] | .status.overall.state == \"green\" end'"
)
with subtest("Metricbeat is running"):
one.wait_for_unit("metricbeat.service")
with subtest("Metricbeat metrics arrive in elasticsearch"):
one.wait_until_succeeds(has_metricbeat() + " | tee /dev/console | grep 'true'")
one.wait_until_succeeds(has_metricbeat())
with subtest("Logstash messages arive in elasticsearch"):
one.wait_until_succeeds(total_hits("flowers") + " | grep -v 0")
one.wait_until_succeeds(total_hits("dragons") + " | grep 0")
one.wait_until_succeeds(expect_hits("flowers"))
one.wait_until_succeeds(expect_no_hits("dragons"))
'' + lib.optionalString (elk ? journalbeat) ''
with subtest(
"A message logged to the journal is ingested by elasticsearch via journalbeat"
):
one.wait_for_unit("journalbeat.service")
one.execute("echo 'Supercalifragilisticexpialidocious' | systemd-cat")
one.wait_until_succeeds(
total_hits("Supercalifragilisticexpialidocious") + " | grep -v 0"
expect_hits("Supercalifragilisticexpialidocious")
)
'' + lib.optionalString (elk ? filebeat) ''
with subtest(
"A message logged to the journal is ingested by elasticsearch via filebeat"
):
one.wait_for_unit("filebeat.service")
one.execute("echo 'Superdupercalifragilisticexpialidocious' | systemd-cat")
one.wait_until_succeeds(
expect_hits("Superdupercalifragilisticexpialidocious")
)
one.execute(
"echo 'SuperdupercalifragilisticexpialidociousIndeed' >> /var/lib/filebeat/test"
)
one.wait_until_succeeds(
expect_hits("SuperdupercalifragilisticexpialidociousIndeed")
)
'' + ''
with subtest("Elasticsearch-curator works"):
one.systemctl("stop logstash")
one.systemctl("start elasticsearch-curator")
one.wait_until_succeeds(
'! curl --silent --show-error "${esUrl}/_cat/indices" | grep logstash | grep ^'
'! curl --silent --show-error --fail-with-body "${esUrl}/_cat/indices" | grep logstash | grep ^'
)
'';
}) { inherit pkgs system; };
@ -235,7 +283,7 @@ in {
# elasticsearch = pkgs.elasticsearch7-oss;
# logstash = pkgs.logstash7-oss;
# kibana = pkgs.kibana7-oss;
# journalbeat = pkgs.journalbeat7;
# filebeat = pkgs.filebeat7;
# metricbeat = pkgs.metricbeat7;
# };
unfree = lib.dontRecurseIntoAttrs {
@ -250,7 +298,7 @@ in {
elasticsearch = pkgs.elasticsearch7;
logstash = pkgs.logstash7;
kibana = pkgs.kibana7;
journalbeat = pkgs.journalbeat7;
filebeat = pkgs.filebeat7;
metricbeat = pkgs.metricbeat7;
};
};

27
nixos/tests/parsedmarc/default.nix
View file

@ -4,6 +4,7 @@
{ pkgs, ... }@args:
let
inherit (import ../../lib/testing-python.nix args) makeTest;
inherit (pkgs) lib;
dmarcTestReport = builtins.fetchurl {
name = "dmarc-test-report";
@ -54,7 +55,7 @@ in
localMail = makeTest
{
name = "parsedmarc-local-mail";
meta = with pkgs.lib.maintainers; {
meta = with lib.maintainers; {
maintainers = [ talyz ];
};
@ -83,7 +84,7 @@ in
};
};
services.elasticsearch.package = pkgs.elasticsearch7-oss;
services.elasticsearch.package = pkgs.elasticsearch-oss;
environment.systemPackages = [
(sendEmail "dmarc@localhost")
@ -94,6 +95,7 @@ in
testScript = { nodes }:
let
esPort = toString nodes.parsedmarc.config.services.elasticsearch.port;
valueObject = lib.optionalString (lib.versionAtLeast nodes.parsedmarc.config.services.elasticsearch.package.version "7") ".value";
in ''
parsedmarc.start()
parsedmarc.wait_for_unit("postfix.service")
@ -104,11 +106,15 @@ in
)
parsedmarc.fail(
"curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
"curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
+ " | tee /dev/console"
+ " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
)
parsedmarc.succeed("send-email")
parsedmarc.wait_until_succeeds(
"curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
"curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
+ " | tee /dev/console"
+ " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
)
'';
};
@ -121,7 +127,7 @@ in
in
makeTest {
name = "parsedmarc-external-mail";
meta = with pkgs.lib.maintainers; {
meta = with lib.maintainers; {
maintainers = [ talyz ];
};
@ -153,7 +159,7 @@ in
};
};
services.elasticsearch.package = pkgs.elasticsearch7-oss;
services.elasticsearch.package = pkgs.elasticsearch-oss;
environment.systemPackages = [
pkgs.jq
@ -201,6 +207,7 @@ in
testScript = { nodes }:
let
esPort = toString nodes.parsedmarc.config.services.elasticsearch.port;
valueObject = lib.optionalString (lib.versionAtLeast nodes.parsedmarc.config.services.elasticsearch.package.version "7") ".value";
in ''
mail.start()
mail.wait_for_unit("postfix.service")
@ -213,11 +220,15 @@ in
)
parsedmarc.fail(
"curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
"curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
+ " | tee /dev/console"
+ " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
)
mail.succeed("send-email")
parsedmarc.wait_until_succeeds(
"curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
"curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
+ " | tee /dev/console"
+ " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
)
'';
};

8
pkgs/development/tools/misc/kibana/6.x.nix
View file

@ -18,12 +18,12 @@ let
shas =
if enableUnfree
then {
x86_64-linux = "1xwklhqxk5rmdrgy2simwvijzq29kyq5w2w3hy53xh2i1zlnyvq3";
x86_64-darwin = "1qpdn28mrpggd55khzqqld6r89l0hb870rigxcw2i8p2yx3jv106";
x86_64-linux = "1a501lavxhckb3l93sbrbqyshicwkk6p89frry4x8p037xcfpy0x";
x86_64-darwin = "0zm45af30shhcg3mdhcma6rms1hyrx62rm5jzwnz9kxv4d30skbw";
}
else {
x86_64-linux = "1wpnwal2rq5v2bsp5qil9j6dplif7ql5394sy4ia5ghp2fzifxmf";
x86_64-darwin = "12z8i0wbw10c097glbpdy350p0h3957433f51qfx2p0ghgkzkhzv";
x86_64-linux = "0wfdipf21apyily7mvlqgyc7m5jpr96zgrryzwa854z3xb2vw8zg";
x86_64-darwin = "1nklfx4yz6hsxlljvnvwjy7pncv9mzngl84710xad5jlyras3sdj";
};
in stdenv.mkDerivation rec {

10
pkgs/development/tools/misc/kibana/7.x.nix
View file

@ -4,23 +4,23 @@
, stdenv
, makeWrapper
, fetchurl
, nodejs-14_x
, nodejs-16_x
, coreutils
, which
}:
with lib;
let
nodejs = nodejs-14_x;
nodejs = nodejs-16_x;
inherit (builtins) elemAt;
info = splitString "-" stdenv.hostPlatform.system;
arch = elemAt info 0;
plat = elemAt info 1;
shas =
{
x86_64-linux = "19p9s4sir982bb1zcldrbphhwfs9i11p0q28vgc421iqg10kjlf1";
x86_64-darwin = "0qq557ngwwakifidyrccga4cadj9k9pzhjwy4msmbcgf5pb86qyc";
aarch64-linux = "183cp1h8d3n7xfcpcys4hf36palczxa409afyp62kzyzckngy0j8";
x86_64-linux = "0jivwsrq31n0qfznrsjfsn65sg3wpbd990afn2wzjnj4drq7plz6";
x86_64-darwin = "02483aqzrccq1x6rwznmcazijdd46yxj9vnbihnvp2xyp3w9as45";
aarch64-linux = "0iw155gkkl1hshc80lfj95rssg039ig21wz1l3srmmf2x4f934s9";
};
in stdenv.mkDerivation rec {

2
pkgs/misc/logging/beats/6.x.nix
View file

@ -8,7 +8,7 @@ let beat = package : extraArgs : buildGoPackage (rec {
owner = "elastic";
repo = "beats";
rev = "v${version}";
sha256 = "0jkiz5dfdi9zsji04ipcmcj7pml9294v455y7s2c22k24gyzbaw8";
sha256 = "1vnw9clsc10cfpjf6vxvc6m507b2q17sgsl079iwqbp4v0286il7";
};
goPackagePath = "github.com/elastic/beats";

24
pkgs/misc/logging/beats/7.x.nix
View file

@ -8,10 +8,10 @@ let beat = package: extraArgs: buildGoModule (rec {
owner = "elastic";
repo = "beats";
rev = "v${version}";
sha256 = "0gjyzprgj9nskvlkm2bf125b7qn3608llz4kh1fyzsvrw6zb7sm8";
sha256 = "sha256-9Jl5Xo1iKdOY9ZE5JXKSL4ee+NdsN3KCY2dDYuxlzPI=";
};
vendorSha256 = "04cwf96fh60ld3ndjzzssgirc9ssb53yq71j6ksx36m3y1x7fq9c";
vendorSha256 = "sha256-tyxyM7RsTHTVVxc9gagPsSvFRaWGTmobKzyv9RODXBk=";
subPackages = [ package ];
@ -24,7 +24,14 @@ let beat = package: extraArgs: buildGoModule (rec {
} // extraArgs);
in
rec {
filebeat7 = beat "filebeat" { meta.description = "Lightweight shipper for logfiles"; };
filebeat7 = beat "filebeat" {
meta.description = "Lightweight shipper for logfiles";
buildInputs = [ systemd ];
tags = [ "withjournald" ];
postFixup = ''
patchelf --set-rpath ${lib.makeLibraryPath [ (lib.getLib systemd) ]} "$out/bin/filebeat"
'';
};
heartbeat7 = beat "heartbeat" { meta.description = "Lightweight shipper for uptime monitoring"; };
metricbeat7 = beat "metricbeat" {
meta.description = "Lightweight shipper for metrics";
@ -47,15 +54,4 @@ rec {
PostgreSQL, Redis or Thrift and correlate the messages into transactions.
'';
};
journalbeat7 = beat "journalbeat" {
meta.description = ''
Journalbeat is an open source data collector to read and forward
journal entries from Linuxes with systemd.
'';
buildInputs = [ systemd.dev ];
postFixup = let libPath = lib.makeLibraryPath [ (lib.getLib systemd) ]; in
''
patchelf --set-rpath ${libPath} "$out/bin/journalbeat"
'';
};
}

4
pkgs/servers/search/elasticsearch/6.x.nix
View file

@ -19,8 +19,8 @@ stdenv.mkDerivation (rec {
url = "https://artifacts.elastic.co/downloads/elasticsearch/${pname}-${version}.tar.gz";
sha256 =
if enableUnfree
then "09dy3iyzk460vra6na6vk7d3mzpbv4cl0pl7kjmybxy947j7hh42"
else "0s04xz3j4psyhawvy503sp2nl5s0gswmpd9wfvwnavgcrr23wk39";
then "1hkcgqsrnnx3zjpgar4424mxfaxrx0zbrp7n7n0dlbhphshwnkmd"
else "1pglg60aigy31xmpfchnxcc04nd18zwc3av4m0kyp00yk5mnlyqm";
};
patches = [ ./es-home-6.x.patch ];

6
pkgs/servers/search/elasticsearch/7.x.nix
View file

@ -18,9 +18,9 @@ let
plat = elemAt info 1;
shas =
{
x86_64-linux = "1ld7656b37l67vi4pyv0il865b168niqnbd4hzbvdnwrm35prp10";
x86_64-darwin = "11b180y11xw5q01l7aw6lyn15lp9ks8xmakjg1j7gp3z6c90hpn3";
aarch64-linux = "0s4ph79x17f90jk31wjwk259dk9dmhnmnkxdcn77m191wvf6m3wy";
x86_64-linux = "1s16l95wc589cr69pfbgmkn9rkvxn6sd6jlbiqpm6p6iyxiaxd6c";
x86_64-darwin = "05h7pvq4pb816wgcymnfklp3w6sv54x6138v2infw5219dnk8pfs";
aarch64-linux = "0q4xnjzhlx1b2lkikca88qh9glfxaifsm419k2bxxlrfrx31zlkq";
};
in
stdenv.mkDerivation rec {

44
pkgs/servers/search/elasticsearch/plugins.nix
View file

@ -38,8 +38,8 @@ in
src = fetchurl {
url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
sha256 =
if version == "7.11.1" then "0mi6fmnjbqypa4n1w34dvlmyq793pz4wf1r5srcs7i84kkiddysy"
else if version == "6.8.3" then "0vbaqyj0lfy3ijl1c9h92b0nh605h5mjs57bk2zhycdvbw5sx2lv"
if version == "7.16.1" then "1sz858m9963xqr5kzjlwnq7k0a146rn60v6xijyfbp8y3brg618p"
else if version == "6.8.21" then "06b1pavyggzfp4wwdql0q9nm3r7i9px9cagp4yh4nhxhnk4w5fiq"
else throw "unsupported version ${version} for plugin ${pluginName}";
};
meta = with lib; {
@ -55,8 +55,8 @@ in
src = fetchurl {
url = "https://github.com/vhyza/elasticsearch-${pluginName}/releases/download/v${version}/elasticsearch-${pluginName}-${version}-plugin.zip";
sha256 =
if version == "7.11.1" then "0r2k2ndgqiqh27lch8dbay1m09f00h5kjcan87chcvyf623l40a3"
else if version == "6.8.3" then "12bshvp01pp2lgwd0cn9l58axg8gdimsh4g9wfllxi1bdpv4cy53"
if version == "7.16.1" then "0yjy9yhw77lmalivxnmv2rq8fk93ddxszkk73lgmpffladx2ikir"
else if version == "6.8.21" then "0m80cn7vkcvk95v4pdmi6vk5ww7p01k0hj2iqb9g870vs6x2qjzv"
else throw "unsupported version ${version} for plugin ${pluginName}";
};
meta = with lib; {
@ -72,8 +72,8 @@ in
src = fetchurl {
url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
sha256 =
if version == "7.11.1" then "10ln81zyf04qi9wv10mck8iz0xwfvwp4ni0hl1gkgvh44lf1n855"
else if version == "6.8.3" then "0ggdhf7w50bxsffmcznrjy14b578fps0f8arg3v54qvj94v9jc37"
if version == "7.16.1" then "1w5ndgffqzj5ijglmykifrk1jsgh7qwn8m7sbpiv0r7n3aayhz1x"
else if version == "6.8.21" then "07w8s4a5gvr9lzjzf629y8rx3kvs6zd1vl07ksw1paghp42yb354"
else throw "unsupported version ${version} for plugin ${pluginName}";
};
meta = with lib; {
@ -89,8 +89,8 @@ in
src = fetchurl {
url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
sha256 =
if version == "7.11.1" then "09grfvqjmm2rznc48z84awh54afh81qa16amfqw3amsb8dr6czm6"
else if version == "6.8.3" then "0pmffz761dqjpvmkl7i7xsyw1iyyspqpddxp89rjsznfc9pak5im"
if version == "7.16.1" then "16mv7b9nl96bcygabvjqidxp2sjk340va19mrmliblpq3mxa2sii"
else if version == "6.8.21" then "1kdpbrasxwr3dn21zjrklp1s389rwa51fairygdwl8px9liwwfa5"
else throw "unsupported version ${version} for plugin ${pluginName}";
};
meta = with lib; {
@ -106,8 +106,8 @@ in
src = fetchurl {
url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
sha256 =
if version == "7.11.1" then "0imkf3w2fmspb78vkf9k6kqx1crm4f82qgnbk1qa7gbsa2j47hbs"
else if version == "6.8.3" then "0kfr4i2rcwinjn31xrc2piicasjanaqcgnbif9xc7lnak2nnzmll"
if version == "7.16.1" then "0bf8f8cybsp6s2ai3j04yay9kbhsafpgxivxjvzn2iy9qgc84ls4"
else if version == "6.8.21" then "0v31yyhjcdlqnjw1f9kihh7z3c6d31whc57hqqd1dn579n4s9rlz"
else throw "unsupported version ${version} for plugin ${pluginName}";
};
meta = with lib; {
@ -123,8 +123,8 @@ in
src = fetchurl {
url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${esVersion}.zip";
sha256 =
if version == "7.11.1" then "0ahyb1plgwvq22id2kcx9g076ybb3kvybwakgcvsdjjdyi4cwgjs"
else if version == "6.8.3" then "1mm6hj2m1db68n81rzsvlw6nisflr5ikzk5zv9nmk0z641n5vh1x"
if version == "7.16.1" then "0sfa0ql3hh8jmha230dyhr51bvsvwmazyycf36ngpmxsysm8ccml"
else if version == "6.8.21" then "0sfh1az30q4f34zxig2fz8wn9gk53fmmxyg5pbi1svn9761p5awq"
else throw "unsupported version ${version} for plugin ${pluginName}";
};
meta = with lib; {
@ -140,8 +140,8 @@ in
src = fetchurl {
url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${esVersion}.zip";
sha256 =
if version == "7.11.1" then "0i98b905k1zwm3y9pfhr40v2fm5qdsp3icygibhxf7drffygk4l7"
else if version == "6.8.3" then "1s2klpvnhpkrk53p64zbga3b66czi7h1a13f58kfn2cn0zfavnbk"
if version == "7.16.1" then "1b95hjr4qhiavm7r7k19bwk5c64r00f1g5s0ydnb6gzym9hdb5s1"
else if version == "6.8.21" then "00lwj00rfdk6850gk1n86chiz2w6afpqn7jn588jdbwv41qh5mrv"
else throw "unsupported version ${version} for plugin ${pluginName}";
};
meta = with lib; {
@ -157,24 +157,24 @@ in
pluginName = "search-guard";
version =
# https://docs.search-guard.com/latest/search-guard-versions
if esVersion == "7.11.1" then "${esVersion}-50.0.0"
else if esVersion == "6.8.3" then "${esVersion}-25.5"
if esVersion == "7.16.1" then "${esVersion}-52.5.0"
else if esVersion == "6.8.21" then "${esVersion}-25.6"
else throw "unsupported version ${esVersion} for plugin ${pluginName}";
src =
if esVersion == "7.11.1" then
if esVersion == "7.16.1" then
fetchurl {
url = "https://maven.search-guard.com/search-guard-suite-release/com/floragunn/search-guard-suite-plugin/${version}/search-guard-suite-plugin-${version}.zip";
sha256 = "1lippygiy0xcxxlakylhvj3bj2i681k6jcfjsprkfk7hlaqsqxkm";
sha256 = "1m3nj35qyrkkh3mhmn66nippavima8h8qpaxddalhjsvf70lhnjb";
}
else if esVersion == "6.8.3" then
else if esVersion == "6.8.21" then
fetchurl {
url = "mirror://maven/com/floragunn/${pluginName}-${majorVersion}/${version}/${pluginName}-${majorVersion}-${version}.zip";
sha256 = "0a7ys9qinc0fjyka03cx9rv0pm7wnvslk234zv5vrphkrj52s1cb";
url = "https://maven.search-guard.com/search-guard-release/com/floragunn/search-guard-6/${version}/search-guard-6-${version}.zip";
sha256 = "19nj513wigwd0mzq747zax4fzvv5vi24f7j0636rydd9iv9cyhg2";
}
else throw "unsupported version ${version} for plugin ${pluginName}";
meta = with lib; {
homepage = "https://search-guard.com";
description = "Elasticsearch plugin that offers encryption, authentication, and authorisation. ";
description = "Elasticsearch plugin that offers encryption, authentication, and authorisation.";
license = licenses.asl20;
};
};

4
pkgs/tools/misc/logstash/6.x.nix
View file

@ -17,8 +17,8 @@ let this = stdenv.mkDerivation rec {
url = "https://artifacts.elastic.co/downloads/logstash/${pname}-${version}.tar.gz";
sha256 =
if enableUnfree
then "00pwi7clgdflzzg15bh3y30gzikvvy7p5fl88fww7xhhy47q8053"
else "0spxgqsyh72n0l0xh6rljp0lbqz46xmr02sqz25ybycr4qkxdhgk";
then "0hij1byw5b3xmk3vshr9p7gxwbjrywr7ylps05ydc2dmnz8q2a79"
else "1fa236pvhj7spys54nqi3k64rwzf6zi6gaccmqg4p4sh92jzsybv";
};
dontBuild = true;

12
pkgs/tools/misc/logstash/7.x.nix
View file

@ -17,14 +17,14 @@ let
shas =
if enableUnfree
then {
x86_64-linux = "0yjaki7gjffrz86hvqgn1gzhd9dc9llcj50g2x1sgpyn88zk0z0p";
x86_64-darwin = "0dqm66c89w1nvmbwqzphlqmf7avrycgv1nwd5b0k1z168fj0c3zm";
aarch64-linux = "11hjhyb48mjagmvqyxb780n57kr619h6p4adl2vs1zm97g9gslx8";
x86_64-linux = "1vm53alq9q1qy2jcsjg9z339xrkac5r9qqpdafp53ny4zsv1n7vj";
x86_64-darwin = "0hhjyl04h3gd66rdk22272rj419br4v2i59lyrmaj6hmnsqbv968";
aarch64-linux = "0yjaki7gjffrz86hvqgn1gzhd9dc9llcj50g2x1sgpyn88zk0z0p";
}
else {
x86_64-linux = "14b1649avjcalcsi0ffkgznq6d93qdk6m3j0i73mwfqka5d3dvy3";
x86_64-darwin = "0ypgdfklr5rxvsnc3czh231pa1z2h70366j1c6q5g64b3xnxpphs";
aarch64-linux = "01ainayr8fwwfix7dmxfhhmb23ji65dn4lbjwnj2w0pl0ym9h9w2";
x86_64-linux = "1f3659vcgczm7v03q3fvsmp1ndp6wm3i7r2b2vbl4xq7hf9v7azk";
x86_64-darwin = "10zw9qc0lc0x9in0nkxc1aiazhyd69l8sya2ni46ivyyjwf0sqsn";
aarch64-linux = "1czhgmky2zf3mqykn5ww4257yfhd36mi4x6dq569ymly83pivf8v";
};
this = stdenv.mkDerivation rec {
version = elk7Version;

1
pkgs/top-level/aliases.nix
View file

@ -384,6 +384,7 @@ mapAliases ({
jbuilder = dune_1; # added 2018-09-09
jikes = throw "jikes was deprecated on 2019-10-07: abandoned by upstream";
joseki = apache-jena-fuseki; # added 2016-02-28
journalbeat7 = throw "journalbeat has been removed upstream. Use filebeat with the journald input instead.";
jvmci8 = throw "graalvm8 and its tools were deprecated in favor of graalvm8-ce"; # added 2021-10-15
json_glib = json-glib; # added 2018-02-25
kafkacat = kcat; # added 2021-10-07

7
pkgs/top-level/all-packages.nix
View file

@ -2140,8 +2140,7 @@ with pkgs;
filebeat7
heartbeat7
metricbeat7
packetbeat7
journalbeat7;
packetbeat7;
filebeat = filebeat6;
heartbeat = heartbeat6;
@ -5000,8 +4999,8 @@ with pkgs;
# The latest version used by elasticsearch, logstash, kibana and the the beats from elastic.
# When updating make sure to update all plugins or they will break!
elk6Version = "6.8.3";
elk7Version = "7.11.1";
elk6Version = "6.8.21";
elk7Version = "7.16.1";
elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix {
util-linux = util-linuxMinimal;