3
0
Fork 0
forked from mirrors/nixpkgs

Add hydra package and its NixOS module.

This was originally removed in d4d0e449d7.
The intent was not to maintain hydra expression at two places.

Nowadays we have enough devs to maintain this despite copy/pasta.

This should encourage more people to use Hydra, which is a really
great piece of software together with Nix.

Tested a deploy using https://github.com/peti/hydra-tutorial
This commit is contained in:
Domen Kožar 2016-06-01 20:02:58 +01:00
parent 344225ce68
commit 3e631800d1
5 changed files with 575 additions and 0 deletions

View file

@ -128,6 +128,7 @@
./services/continuous-integration/jenkins/default.nix
./services/continuous-integration/jenkins/slave.nix
./services/continuous-integration/jenkins/job-builder.nix
./services/continuous-integration/hydra/default.nix
./services/databases/4store-endpoint.nix
./services/databases/4store.nix
./services/databases/couchdb.nix

View file

@ -0,0 +1,418 @@
{ config, pkgs, lib, ... }:
with lib;
let
cfg = config.services.hydra;
baseDir = "/var/lib/hydra";
hydraConf = pkgs.writeScript "hydra.conf" cfg.extraConfig;
hydraEnv =
{ HYDRA_DBI = cfg.dbi;
HYDRA_CONFIG = "${baseDir}/hydra.conf";
HYDRA_DATA = "${baseDir}";
};
env =
{ NIX_REMOTE = "daemon";
SSL_CERT_FILE = "/etc/ssl/certs/ca-certificates.crt"; # Remove in 16.03
PGPASSFILE = "${baseDir}/pgpass";
NIX_REMOTE_SYSTEMS = concatStringsSep ":" cfg.buildMachinesFiles;
} // optionalAttrs (cfg.smtpHost != null) {
EMAIL_SENDER_TRANSPORT = "SMTP";
EMAIL_SENDER_TRANSPORT_host = cfg.smtpHost;
} // hydraEnv // cfg.extraEnv;
serverEnv = env //
{ HYDRA_TRACKER = cfg.tracker;
COLUMNS = "80";
PGPASSFILE = "${baseDir}/pgpass-www"; # grrr
} // (optionalAttrs cfg.debugServer { DBIC_TRACE = "1"; });
localDB = "dbi:Pg:dbname=hydra;user=hydra;";
haveLocalDB = cfg.dbi == localDB;
in
{
###### interface
options = {
services.hydra = rec {
enable = mkOption {
type = types.bool;
default = false;
description = ''
Whether to run Hydra services.
'';
};
dbi = mkOption {
type = types.str;
default = localDB;
example = "dbi:Pg:dbname=hydra;host=postgres.example.org;user=foo;";
description = ''
The DBI string for Hydra database connection.
'';
};
package = mkOption {
type = types.path;
default = pkgs.hydra;
defaultText = "pkgs.hydra";
description = "The Hydra package.";
};
hydraURL = mkOption {
type = types.str;
description = ''
The base URL for the Hydra webserver instance. Used for links in emails.
'';
};
listenHost = mkOption {
type = types.str;
default = "*";
example = "localhost";
description = ''
The hostname or address to listen on or <literal>*</literal> to listen
on all interfaces.
'';
};
port = mkOption {
type = types.int;
default = 3000;
description = ''
TCP port the web server should listen to.
'';
};
minimumDiskFree = mkOption {
type = types.int;
default = 0;
description = ''
Threshold of minimum disk space (GiB) to determine if the queue runner should run or not.
'';
};
minimumDiskFreeEvaluator = mkOption {
type = types.int;
default = 0;
description = ''
Threshold of minimum disk space (GiB) to determine if the evaluator should run or not.
'';
};
notificationSender = mkOption {
type = types.str;
description = ''
Sender email address used for email notifications.
'';
};
smtpHost = mkOption {
type = types.nullOr types.str;
default = null;
example = ["localhost"];
description = ''
Hostname of the SMTP server to use to send email.
'';
};
tracker = mkOption {
type = types.str;
default = "";
description = ''
Piece of HTML that is included on all pages.
'';
};
logo = mkOption {
type = types.nullOr types.path;
default = null;
description = ''
Path to a file containing the logo of your Hydra instance.
'';
};
debugServer = mkOption {
type = types.bool;
default = false;
description = "Whether to run the server in debug mode.";
};
extraConfig = mkOption {
type = types.lines;
description = "Extra lines for the Hydra configuration.";
};
extraEnv = mkOption {
type = types.attrsOf types.str;
default = {};
description = "Extra environment variables for Hydra.";
};
gcRootsDir = mkOption {
type = types.path;
default = "/nix/var/nix/gcroots/hydra";
description = "Directory that holds Hydra garbage collector roots.";
};
buildMachinesFiles = mkOption {
type = types.listOf types.path;
default = [];
example = [ "/etc/nix/machines" "/var/lib/hydra/provisioner/machines" ];
description = "List of files containing build machines.";
};
useSubstitutes = mkOption {
type = types.bool;
default = false;
description = ''
Whether to use binary caches for downloading store paths. Note that
binary substitutions trigger (a potentially large number of) additional
HTTP requests that slow down the queue monitor thread significantly.
Also, this Hydra instance will serve those downloaded store paths to
its users with its own signature attached as if it had built them
itself, so don't enable this feature unless your active binary caches
are absolute trustworthy.
'';
};
};
};
###### implementation
config = mkIf cfg.enable {
users.extraGroups.hydra = { };
users.extraUsers.hydra =
{ description = "Hydra";
group = "hydra";
createHome = true;
home = baseDir;
useDefaultShell = true;
};
users.extraUsers.hydra-queue-runner =
{ description = "Hydra queue runner";
group = "hydra";
useDefaultShell = true;
home = "${baseDir}/queue-runner"; # really only to keep SSH happy
};
users.extraUsers.hydra-www =
{ description = "Hydra web server";
group = "hydra";
useDefaultShell = true;
};
nix.trustedUsers = [ "hydra-queue-runner" ];
services.hydra.extraConfig =
''
using_frontend_proxy 1
base_uri ${cfg.hydraURL}
notification_sender ${cfg.notificationSender}
max_servers 25
${optionalString (cfg.logo != null) ''
hydra_logo ${cfg.logo}
''}
gc_roots_dir ${cfg.gcRootsDir}
'';
environment.systemPackages = [ cfg.package ];
environment.variables = hydraEnv;
nix.extraOptions = ''
gc-keep-outputs = true
gc-keep-derivations = true
# The default (`true') slows Nix down a lot since the build farm
# has so many GC roots.
gc-check-reachability = false
'';
systemd.services.hydra-init =
{ wantedBy = [ "multi-user.target" ];
requires = optional haveLocalDB "postgresql.service";
after = optional haveLocalDB "postgresql.service";
environment = env;
preStart = ''
mkdir -p ${baseDir}
chown hydra.hydra ${baseDir}
chmod 0750 ${baseDir}
ln -sf ${hydraConf} ${baseDir}/hydra.conf
mkdir -m 0700 -p ${baseDir}/www
chown hydra-www.hydra ${baseDir}/www
mkdir -m 0700 -p ${baseDir}/queue-runner
mkdir -m 0750 -p ${baseDir}/build-logs
chown hydra-queue-runner.hydra ${baseDir}/queue-runner ${baseDir}/build-logs
${optionalString haveLocalDB ''
if ! [ -e ${baseDir}/.db-created ]; then
${config.services.postgresql.package}/bin/createuser hydra
${config.services.postgresql.package}/bin/createdb -O hydra hydra
touch ${baseDir}/.db-created
fi
''}
if [ ! -e ${cfg.gcRootsDir} ]; then
# Move legacy roots directory.
if [ -e /nix/var/nix/gcroots/per-user/hydra/hydra-roots ]; then
mv /nix/var/nix/gcroots/per-user/hydra/hydra-roots ${cfg.gcRootsDir}
fi
mkdir -p ${cfg.gcRootsDir}
fi
# Move legacy hydra-www roots.
if [ -e /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots ]; then
find /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots/ -type f \
| xargs -r mv -f -t ${cfg.gcRootsDir}/
rmdir /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots
fi
chown hydra.hydra ${cfg.gcRootsDir}
chmod 2775 ${cfg.gcRootsDir}
'';
serviceConfig.ExecStart = "${cfg.package}/bin/hydra-init";
serviceConfig.PermissionsStartOnly = true;
serviceConfig.User = "hydra";
serviceConfig.Type = "oneshot";
serviceConfig.RemainAfterExit = true;
};
systemd.services.hydra-server =
{ wantedBy = [ "multi-user.target" ];
requires = [ "hydra-init.service" ];
after = [ "hydra-init.service" ];
environment = serverEnv;
serviceConfig =
{ ExecStart =
"@${cfg.package}/bin/hydra-server hydra-server -f -h '${cfg.listenHost}' "
+ "-p ${toString cfg.port} --max_spare_servers 5 --max_servers 25 "
+ "--max_requests 100 ${optionalString cfg.debugServer "-d"}";
User = "hydra-www";
PermissionsStartOnly = true;
Restart = "always";
};
};
systemd.services.hydra-queue-runner =
{ wantedBy = [ "multi-user.target" ];
requires = [ "hydra-init.service" ];
after = [ "hydra-init.service" "network.target" ];
path = [ cfg.package pkgs.nettools pkgs.openssh pkgs.bzip2 config.nix.package ];
environment = env // {
PGPASSFILE = "${baseDir}/pgpass-queue-runner"; # grrr
IN_SYSTEMD = "1"; # to get log severity levels
};
serviceConfig =
{ ExecStart = "@${cfg.package}/bin/hydra-queue-runner hydra-queue-runner -v --option build-use-substitutes ${if cfg.useSubstitutes then "true" else "false"}";
ExecStopPost = "${cfg.package}/bin/hydra-queue-runner --unlock";
User = "hydra-queue-runner";
Restart = "always";
# Ensure we can get core dumps.
LimitCORE = "infinity";
WorkingDirectory = "${baseDir}/queue-runner";
};
};
systemd.services.hydra-evaluator =
{ wantedBy = [ "multi-user.target" ];
requires = [ "hydra-init.service" ];
after = [ "hydra-init.service" "network.target" ];
path = [ pkgs.nettools ];
environment = env;
serviceConfig =
{ ExecStart = "@${cfg.package}/bin/hydra-evaluator hydra-evaluator";
User = "hydra";
Restart = "always";
WorkingDirectory = baseDir;
};
};
systemd.services.hydra-update-gc-roots =
{ requires = [ "hydra-init.service" ];
after = [ "hydra-init.service" ];
environment = env;
serviceConfig =
{ ExecStart = "@${cfg.package}/bin/hydra-update-gc-roots hydra-update-gc-roots";
User = "hydra";
};
startAt = "2,14:15";
};
systemd.services.hydra-send-stats =
{ wantedBy = [ "multi-user.target" ];
after = [ "hydra-init.service" ];
environment = env;
serviceConfig =
{ ExecStart = "@${cfg.package}/bin/hydra-send-stats hydra-send-stats";
User = "hydra";
};
};
# If there is less than a certain amount of free disk space, stop
# the queue/evaluator to prevent builds from failing or aborting.
systemd.services.hydra-check-space =
{ script =
''
if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFree} * 1024**3)) ]; then
echo "stopping Hydra queue runner due to lack of free space..."
systemctl stop hydra-queue-runner
fi
if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFreeEvaluator} * 1024**3)) ]; then
echo "stopping Hydra evaluator due to lack of free space..."
systemctl stop hydra-evaluator
fi
'';
startAt = "*:0/5";
};
# Periodically compress build logs. The queue runner compresses
# logs automatically after a step finishes, but this doesn't work
# if the queue runner is stopped prematurely.
systemd.services.hydra-compress-logs =
{ path = [ pkgs.bzip2 ];
script =
''
find /var/lib/hydra/build-logs -type f -name "*.drv" -mtime +3 -size +0c | xargs -r bzip2 -v -f
'';
startAt = "Sun 01:45";
};
services.postgresql.enable = mkIf haveLocalDB true;
services.postgresql.identMap = optionalString haveLocalDB
''
hydra-users hydra hydra
hydra-users hydra-queue-runner hydra
hydra-users hydra-www hydra
hydra-users root hydra
'';
services.postgresql.authentication = optionalString haveLocalDB
''
local hydra all ident map=hydra-users
'';
};
}

View file

@ -0,0 +1,142 @@
{ stdenv, nixUnstable, perlPackages, buildEnv, releaseTools, fetchFromGitHub
, makeWrapper, autoconf, automake, libtool, unzip, pkgconfig, sqlite, libpqxx
, gitAndTools, mercurial, darcs, subversion, bazaar, openssl, bzip2, libxslt
, guile, perl, postgresql92, aws-sdk-cpp, nukeReferences, git, boehmgc
, docbook_xsl, openssh, gnused, coreutils, findutils, gzip, lzma, gnutar
, rpm, dpkg, cdrkit }:
with stdenv;
let
perlDeps = buildEnv {
name = "hydra-perl-deps";
paths = with perlPackages;
[ ModulePluggable
CatalystActionREST
CatalystAuthenticationStoreDBIxClass
CatalystDevel
CatalystDispatchTypeRegex
CatalystPluginAccessLog
CatalystPluginAuthorizationRoles
CatalystPluginCaptcha
CatalystPluginSessionStateCookie
CatalystPluginSessionStoreFastMmap
CatalystPluginStackTrace
CatalystPluginUnicodeEncoding
CatalystTraitForRequestProxyBase
CatalystViewDownload
CatalystViewJSON
CatalystViewTT
CatalystXScriptServerStarman
CryptRandPasswd
DBDPg
DBDSQLite
DataDump
DateTime
DigestSHA1
EmailMIME
EmailSender
FileSlurp
IOCompress
IPCRun
JSONXS
LWP
LWPProtocolHttps
NetAmazonS3
NetStatsd
PadWalker
Readonly
SQLSplitStatement
SetScalar
Starman
SysHostnameLong
TestMore
TextDiff
TextTable
XMLSimple
nixUnstable
git
boehmgc
];
};
in releaseTools.nixBuild rec {
name = "hydra-${version}";
version = "2016-04-15";
src = fetchFromGitHub {
owner = "NixOS";
repo = "hydra";
rev = "177bf25d648092826a75369191503a3f2bb763a4";
sha256 = "0ngipzm2i2vz5ygfd70hh82d027snpl85r8ncn1rxlkak0g8fxsl";
};
buildInputs =
[ makeWrapper autoconf automake libtool unzip nukeReferences pkgconfig sqlite libpqxx
gitAndTools.topGit mercurial darcs subversion bazaar openssl bzip2 libxslt
guile # optional, for Guile + Guix support
perlDeps perl nixUnstable
postgresql92 # for running the tests
(lib.overrideDerivation (aws-sdk-cpp.override {
apis = ["s3"];
customMemoryManagement = false;
}) (attrs: {
src = fetchFromGitHub {
owner = "edolstra";
repo = "aws-sdk-cpp";
rev = "local";
sha256 = "1vhgsxkhpai9a7dk38q4r239l6dsz2jvl8hii24c194lsga3g84h";
};
}))
];
hydraPath = lib.makeBinPath (
[ libxslt sqlite subversion openssh nixUnstable coreutils findutils
gzip bzip2 lzma gnutar unzip git gitAndTools.topGit mercurial darcs gnused bazaar
] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] );
postUnpack = ''
# Clean up when building from a working tree.
(cd $sourceRoot && (git ls-files -o --directory | xargs -r rm -rfv)) || true
'';
configureFlags = [ "--with-docbook-xsl=${docbook_xsl}/xml/xsl/docbook" ];
preHook = ''
PATH=$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH
PERL5LIB=$(pwd)/src/lib:$PERL5LIB;
'';
preConfigure = "autoreconf -vfi";
enableParallelBuilding = true;
preCheck = ''
patchShebangs .
export LOGNAME=${LOGNAME:-foo}
'';
postInstall = ''
mkdir -p $out/nix-support
for i in $out/bin/*; do
read -n 4 chars < $i
if [[ $chars =~ ELF ]]; then continue; fi
wrapProgram $i \
--prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \
--prefix PATH ':' $out/bin:$hydraPath \
--set HYDRA_RELEASE ${version} \
--set HYDRA_HOME $out/libexec/hydra \
--set NIX_RELEASE ${nixUnstable.name or "unknown"}
done
''; # */
dontStrip = true;
passthru.perlDeps = perlDeps;
meta = with stdenv.lib; {
description = "Nix-based continuous build system";
license = licenses.gpl3;
platforms = platforms.linux;
maintainers = with maintainers; [ domenkozar ];
};
}

View file

@ -7373,6 +7373,8 @@ in
hwloc = callPackage ../development/libraries/hwloc {};
hydra = callPackage ../development/tools/misc/hydra {};
hydraAntLogger = callPackage ../development/libraries/java/hydra-ant-logger { };
hyena = callPackage ../development/libraries/hyena { };

View file

@ -9444,6 +9444,18 @@ let self = _self // overrides; _self = with self; {
};
};
NetStatsd = buildPerlPackage {
name = "Net-Statsd-0.11";
src = fetchurl {
url = mirror://cpan/authors/id/C/CO/COSIMO/Net-Statsd-0.11.tar.gz;
sha256 = "0f56c95846c7e65e6d32cec13ab9df65716429141f106d2dc587f1de1e09e163";
};
meta = {
description = "Sends statistics to the stats daemon over UDP";
license = "perl";
};
};
NetTwitterLite = buildPerlPackage {
name = "Net-Twitter-Lite-0.11002";
src = fetchurl {