Merge pull request #51225 from LumiGuide/elk-6.5.1

elk: 6.3.2 -> 6.5.1

nixos/modules/services/search/kibana.nix

@@ -149,7 +149,10 @@ in {
       after = [ "network.target" "elasticsearch.service" ];
       environment = { BABEL_CACHE_PATH = "${cfg.dataDir}/.babelcache.json"; };
       serviceConfig = {
-        ExecStart = "${cfg.package}/bin/kibana --config ${cfgFile}";
+        ExecStart =
+          "${cfg.package}/bin/kibana" +
+          " --config ${cfgFile}" +
+          " --path.data ${cfg.dataDir}";
         User = "kibana";
         WorkingDirectory = cfg.dataDir;
       };

pkgs/development/tools/misc/kibana/default.nix

@@ -18,12 +18,12 @@ let
   shas =
     if enableUnfree
     then {
-      "x86_64-linux"  = "1kk97ggpzmblhqm6cfd2sv5940f58h323xcyg6rba1njj7lzanv0";
-      "x86_64-darwin" = "1xvwffk8d8br92h0laf4b1m76kvki6cj0pbgcvirfcj1r70vk6c3";
+      "x86_64-linux"  = "0lip4bj3jazv83gydw99dnp03cb0fd1p4z3lvpjbisgmqffbbg5v";
+      "x86_64-darwin" = "0hjdnqagcwbjhpcfyr6w0zmy4sjnx4fyp79czb0vp7dig5arnwm3";
     }
     else {
-      "x86_64-linux"  = "0m81ki1v61gpwb3s6zf84azqrirlm9pdfx65g3xmvdp3d3wii5ly";
-      "x86_64-darwin" = "0zh9p6vsq1d0gh6ks7z6bh8sbhn6rm4jshjcfp3c9k7n2qa8vv9b";
+      "x86_64-linux"  = "1jybn4q7pz61iijzl85d948szlacfcbldn2nhhsb6063xwvf30sa";
+      "x86_64-darwin" = "1bl1h6hgp9l5cjq6pzj2x855wjaka8hbs0fn2c03lbzsc991dppr";
     };
 
   # For the correct phantomjs version see:
@@ -47,6 +47,13 @@ in stdenv.mkDerivation rec {
     sha256 = shas."${stdenv.hostPlatform.system}" or (throw "Unknown architecture");
   };
 
+  patches = [
+    # Kibana specifies it specifically needs nodejs 8.11.4 but nodejs in nixpkgs is at 8.12.0.
+    # The <nixpkgs/nixos/tests/elk.nix> test succeeds with this newer version so lets just
+    # disable the version check.
+    ./disable-nodejs-version-check.patch
+  ];
+
   buildInputs = [ makeWrapper ];
 
   installPhase = ''

pkgs/development/tools/misc/kibana/disable-nodejs-version-check.patch

@@ -0,0 +1,19 @@
+diff -Naur a/src/setup_node_env/node_version_validator.js b/src/setup_node_env/node_version_validator.js
+--- a/src/setup_node_env/node_version_validator.js	2018-11-16 03:28:42.000000000 +0100
++++ b/src/setup_node_env/node_version_validator.js	2018-12-01 12:19:48.238337176 +0100
+@@ -26,7 +26,7 @@
+ var currentVersion = process && process.version || null;
+ var rawRequiredVersion = pkg && pkg.engines && pkg.engines.node || null;
+ var requiredVersion = rawRequiredVersion ? 'v' + rawRequiredVersion : rawRequiredVersion;
+-var isVersionValid = !!currentVersion && !!requiredVersion && currentVersion === requiredVersion;
++var isVersionValid = !!currentVersion && !!requiredVersion;
+ 
+ // Validates current the NodeJS version compatibility when Kibana starts.
+ if (!isVersionValid) {
+@@ -35,4 +35,4 @@
+   // Actions to apply when validation fails: error report + exit.
+   console.error(errorMessage);
+   process.exit(1);
+-}
+\ No newline at end of file
++}

pkgs/misc/logging/beats/6.x.nix

@@ -1,4 +1,4 @@
-{ stdenv, fetchFromGitHub, elk6Version, buildGoPackage, libpcap }:
+{ stdenv, fetchFromGitHub, elk6Version, buildGoPackage, libpcap, systemd }:
 
 let beat = package : extraArgs : buildGoPackage (rec {
       name = "${package}-${version}";
@@ -8,7 +8,7 @@ let beat = package : extraArgs : buildGoPackage (rec {
         owner = "elastic";
         repo = "beats";
         rev = "v${version}";
-        sha256 = "0ymg6y6v0mdhs1rs11fn33xdp3r6v85563z0f4p7s22j1kd3nd6r";
+        sha256 = "1qnrq9bhk7csgcxycb8c7975lq0p7cxw29i6sji777zv4hn7442m";
       };
 
       goPackagePath = "github.com/elastic/beats";
@@ -39,4 +39,11 @@ in {
       PostgreSQL, Redis or Thrift and correlate the messages into transactions.
     '';
   };
+  journalbeat6  = beat "journalbeat" {
+    meta.description = ''
+      Journalbeat is an open source data collector to read and forward
+      journal entries from Linuxes with systemd.
+    '';
+    buildInputs = [ systemd.dev ];
+  };
 }

pkgs/servers/search/elasticsearch/default.nix

@@ -19,14 +19,20 @@ stdenv.mkDerivation (rec {
     url = "https://artifacts.elastic.co/downloads/elasticsearch/${name}.tar.gz";
     sha256 =
       if enableUnfree
-      then "0960ak602pm95p2mha9cb1mrwdky8pfw3y89r2v4zpr5n730hmnh"
-      else "1i4i1ai75bf8k0zd1qf8x0bavrm8rcw13xdim443zza09w95ypk4";
+      then "096i8xiy7mfwlslym9mkjb2f5vqdcqhk65583526rcybqxc2zkqp"
+      else "0j3q02c4rw8272w07hm64sk5ssmj4gj8s3qigsbrq5pgf8b03fvs";
   };
 
   patches = [ ./es-home-6.x.patch ];
 
   postPatch = ''
-    sed -i "s|ES_CLASSPATH=\"\$ES_HOME/lib/\*\"|ES_CLASSPATH=\"$out/lib/*\"|" ./bin/elasticsearch-env
+    substituteInPlace bin/elasticsearch-env --replace \
+      "ES_CLASSPATH=\"\$ES_HOME/lib/\*\"" \
+      "ES_CLASSPATH=\"$out/lib/*\""
+
+    substituteInPlace bin/elasticsearch-cli --replace \
+      "ES_CLASSPATH=\"\$ES_CLASSPATH:\$ES_HOME/\$additional_classpath_directory/\*\"" \
+      "ES_CLASSPATH=\"\$ES_CLASSPATH:$out/\$additional_classpath_directory/\*\""
   '';
 
   buildInputs = [ makeWrapper jre_headless utillinux ]

pkgs/servers/search/elasticsearch/plugins.nix

@@ -27,7 +27,7 @@ in {
     version = "${elk6Version}";
     src = fetchurl {
       url = "https://github.com/vhyza/elasticsearch-analysis-lemmagen/releases/download/v${version}/${name}-plugin.zip";
-      sha256 = "1m4z05wixjrq4nlbdjyhvprkrwxjym8aba18scmzfn25fhbjgvkz";
+      sha256 = "0299ldqwjn1gn44yyjiqjrxvs6mlclhzl1dbn6xlgg1a2lkaal4v";
     };
     meta = with stdenv.lib; {
       homepage = https://github.com/vhyza/elasticsearch-analysis-lemmagen;
@@ -42,7 +42,7 @@ in {
     version = "${elk6Version}";
     src = pkgs.fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/discovery-ec2/discovery-ec2-${elk6Version}.zip";
-      sha256 = "1i7ksy69132sr84h51lamgq967yz3a3dw0b54nckxpqwad9pcpj0";
+      sha256 = "1mg9knbc4r21kaiqnmkd8nzf2i23w5zxqnxyz484q0l2jf4hlkq1";
     };
     meta = with stdenv.lib; {
       homepage = https://github.com/elastic/elasticsearch/tree/master/plugins/discovery-ec2;
@@ -54,10 +54,10 @@ in {
   search_guard = esPlugin rec {
     name = "elastic-search-guard-${version}";
     pluginName = "search-guard";
-    version = "${elk6Version}-22.3";
+    version = "${elk6Version}-23.2";
     src = fetchurl rec {
       url = "mirror://maven/com/floragunn/search-guard-6/${version}/search-guard-6-${version}.zip";
-      sha256 = "1r71h4h9bmxak1mq5gpm19xq5ji1gry1kp3sjmm8azy4ykdqdncx";
+      sha256 = "05310wyxzhylxr0dfgzr10pb0pak30ry8r97g49n6iqj8dw3csnb";
     };
     meta = with stdenv.lib; {
       homepage = https://github.com/floragunncom/search-guard;

pkgs/tools/misc/logstash/default.nix

@@ -16,8 +16,8 @@ stdenv.mkDerivation rec {
     url = "https://artifacts.elastic.co/downloads/logstash/${name}.tar.gz";
     sha256 =
       if enableUnfree
-      then "0yx9hpiav4d5z1b52x2h5i0iknqs9lmxy8vmz0wkb23mjiz8njdr"
-      else "1ir8pbq706mxr56k5cgc9ajn2jp603zrqj66dimx6xxf2nfamw0w";
+      then "01mkb9fr63m3ilp4cbbjccid5m8yc7iqhnli12ynfabsf7302fdz"
+      else "0r60183yyywabinsv9pkd8sx0wq68h740xi3172fypjfdcqs0g9c";
   };
 
   dontBuild         = true;

pkgs/top-level/all-packages.nix

@@ -921,7 +921,8 @@ with pkgs;
     filebeat6
     heartbeat6
     metricbeat6
-    packetbeat6;
+    packetbeat6
+    journalbeat6;
 
   filebeat = filebeat6;
   heartbeat = heartbeat6;
@@ -2455,7 +2456,7 @@ with pkgs;
 
   # The latest version used by elasticsearch, logstash, kibana and the the beats from elastic.
   elk5Version = "5.6.9";
-  elk6Version = "6.3.2";
+  elk6Version = "6.5.1";
 
   elasticsearch5 = callPackage ../servers/search/elasticsearch/5.x.nix { };
   elasticsearch6 = callPackage ../servers/search/elasticsearch { };