3
0
Fork 0
forked from mirrors/nixpkgs

nixos/borgbackup: init

This commit is contained in:
Robert Schütz 2018-03-12 20:20:24 +01:00
parent 6fa864ed7f
commit fdf0f037be
3 changed files with 733 additions and 11 deletions

View file

@ -159,6 +159,7 @@
./services/audio/ympd.nix
./services/backup/almir.nix
./services/backup/bacula.nix
./services/backup/borgbackup.nix
./services/backup/crashplan.nix
./services/backup/crashplan-small-business.nix
./services/backup/mysql-backup.nix

View file

@ -0,0 +1,580 @@
{ config, lib, pkgs, ... }:
with lib;
let
isLocalPath = x:
builtins.substring 0 1 x == "/" # absolute path
|| builtins.substring 0 1 x == "." # relative path
|| builtins.match "[.*:.*]" == null; # not machine:path
mkExcludeFile = cfg:
# Write each exclude pattern to a new line
pkgs.writeText "excludefile" (concatStringsSep "\n" cfg.exclude);
mkKeepArgs = cfg:
# If cfg.prune.keep e.g. has a yearly attribute,
# its content is passed on as --keep-yearly
concatStringsSep " "
(mapAttrsToList (x: y: "--keep-${x}=${toString y}") cfg.prune.keep);
mkBackupScript = cfg: ''
on_exit()
{
exitStatus=$?
# Reset the EXIT handler, or else we're called again on 'exit' below
trap - EXIT
${cfg.postHook}
exit $exitStatus
}
trap 'on_exit' INT TERM QUIT EXIT
archiveName="${cfg.archiveBaseName}-$(date ${cfg.dateFormat})"
archiveSuffix="${optionalString cfg.appendFailedSuffix ".failed"}"
${cfg.preHook}
'' + optionalString cfg.doInit ''
# Run borg init if the repo doesn't exist yet
if ! borg list > /dev/null; then
borg init \
--encryption ${cfg.encryption.mode} \
$extraInitArgs
${cfg.postInit}
fi
'' + ''
borg create \
--compression ${cfg.compression} \
--exclude-from ${mkExcludeFile cfg} \
$extraCreateArgs \
"::$archiveName$archiveSuffix" \
${escapeShellArgs cfg.paths}
'' + optionalString cfg.appendFailedSuffix ''
borg rename "::$archiveName$archiveSuffix" "$archiveName"
'' + ''
${cfg.postCreate}
'' + optionalString (cfg.prune.keep != { }) ''
borg prune \
${mkKeepArgs cfg} \
--prefix ${escapeShellArg cfg.prune.prefix} \
$extraPruneArgs
${cfg.postPrune}
'';
mkPassEnv = cfg: with cfg.encryption;
if passCommand != null then
{ BORG_PASSCOMMAND = passCommand; }
else if passphrase != null then
{ BORG_PASSPHRASE = passphrase; }
else { };
mkBackupService = name: cfg:
let
userHome = config.users.users.${cfg.user}.home;
in nameValuePair "borgbackup-job-${name}" {
description = "BorgBackup job ${name}";
path = with pkgs; [
borgbackup openssh
];
script = mkBackupScript cfg;
serviceConfig = {
User = cfg.user;
Group = cfg.group;
# Only run when no other process is using CPU or disk
CPUSchedulingPolicy = "idle";
IOSchedulingClass = "idle";
ProtectSystem = "strict";
ReadWritePaths =
[ "${userHome}/.config/borg" "${userHome}/.cache/borg" ]
# Borg needs write access to repo if it is not remote
++ optional (isLocalPath cfg.repo) cfg.repo;
PrivateTmp = true;
};
environment = {
BORG_REPO = cfg.repo;
inherit (cfg) extraInitArgs extraCreateArgs extraPruneArgs;
} // (mkPassEnv cfg) // cfg.environment;
inherit (cfg) startAt;
};
# Paths listed in ReadWritePaths must exist before service is started
mkActivationScript = name: cfg:
let
install = "install -o ${cfg.user} -g ${cfg.group}";
in
nameValuePair "borgbackup-job-${name}" (stringAfter [ "users" ] (''
# Eensure that the home directory already exists
# We can't assert createHome == true because that's not the case for root
cd "${config.users.users.${cfg.user}.home}"
${install} -d .config/borg
${install} -d .cache/borg
'' + optionalString (isLocalPath cfg.repo) ''
${install} -d ${escapeShellArg cfg.repo}
''));
mkPassAssertion = name: cfg: {
assertion = with cfg.encryption;
mode != "none" -> passCommand != null || passphrase != null;
message =
"passCommand or passphrase has to be specified because"
+ '' borgbackup.jobs.${name}.encryption != "none"'';
};
mkRepoService = name: cfg:
nameValuePair "borgbackup-repo-${name}" {
description = "Create BorgBackup repository ${name} directory";
script = ''
mkdir -p ${escapeShellArg cfg.path}
chown ${cfg.user}:${cfg.group} ${escapeShellArg cfg.path}
'';
serviceConfig = {
# The service's only task is to ensure that the specified path exists
Type = "oneshot";
};
wantedBy = [ "multi-user.target" ];
};
mkAuthorizedKey = cfg: appendOnly: key:
let
# Because of the following line, clients do not need to specify an absolute repo path
cdCommand = "cd ${escapeShellArg cfg.path}";
restrictedArg = "--restrict-to-${if cfg.allowSubRepos then "path" else "repository"} .";
appendOnlyArg = optionalString appendOnly "--append-only";
quotaArg = optionalString (cfg.quota != null) "--storage-quota ${cfg.quota}";
serveCommand = "borg serve ${restrictedArg} ${appendOnlyArg} ${quotaArg}";
in
''command="${cdCommand} && ${serveCommand}",restrict ${key}'';
mkUsersConfig = name: cfg: {
users.${cfg.user} = {
openssh.authorizedKeys.keys =
(map (mkAuthorizedKey cfg false) cfg.authorizedKeys
++ map (mkAuthorizedKey cfg true) cfg.authorizedKeysAppendOnly);
useDefaultShell = true;
};
groups.${cfg.group} = { };
};
mkKeysAssertion = name: cfg: {
assertion = cfg.authorizedKeys != [ ] || cfg.authorizedKeysAppendOnly != [ ];
message =
"borgbackup.repos.${name} does not make sense"
+ " without at least one public key";
};
in {
meta.maintainers = with maintainers; [ dotlambda ];
###### interface
options.services.borgbackup.jobs = mkOption {
description = "Deduplicating backups using BorgBackup.";
default = { };
example = literalExample ''
{
rootBackup = {
paths = "/";
exclude = [ "/nix" ];
repo = "/path/to/local/repo";
encryption = {
mode = "repokey";
passphrase = "secret";
};
compression = "auto,lzma";
startAt = "weekly";
};
}
'';
type = types.attrsOf (types.submodule (let globalConfig = config; in
{ name, config, ... }: {
options = {
paths = mkOption {
type = with types; either path (nonEmptyListOf path);
description = "Path(s) to back up.";
example = "/home/user";
apply = x: if isList x then x else [ x ];
};
repo = mkOption {
type = types.str;
description = "Remote or local repository to back up to.";
example = "user@machine:/path/to/repo";
};
archiveBaseName = mkOption {
type = types.strMatching "[^/{}]+";
default = "${globalConfig.networking.hostName}-${name}";
defaultText = "\${config.networking.hostName}-<name>";
description = ''
How to name the created archives. A timestamp, whose format is
determined by <option>dateFormat</option>, will be appended. The full
name can be modified at runtime (<literal>$archiveName</literal>).
Placeholders like <literal>{hostname}</literal> must not be used.
'';
};
dateFormat = mkOption {
type = types.str;
description = ''
Arguments passed to <command>date</command>
to create a timestamp suffix for the archive name.
'';
default = "+%Y-%m-%dT%H:%M:%S";
example = "-u +%s";
};
startAt = mkOption {
type = with types; either str (listOf str);
default = "daily";
description = ''
When or how often the backup should run.
Must be in the format described in
<citerefentry><refentrytitle>systemd.time</refentrytitle>
<manvolnum>7</manvolnum></citerefentry>.
If you do not want the backup to start
automatically, use <literal>[ ]</literal>.
'';
};
user = mkOption {
type = types.str;
description = ''
The user <command>borg</command> is run as.
User or group need read permission
for the specified <option>paths</option>.
'';
default = "root";
};
group = mkOption {
type = types.str;
description = ''
The group borg is run as. User or group needs read permission
for the specified <option>paths</option>.
'';
default = "root";
};
encryption.mode = mkOption {
type = types.enum [
"repokey" "keyfile"
"repokey-blake2" "keyfile-blake2"
"authenticated" "authenticated-blake2"
"none"
];
description = ''
Encryption mode to use. Setting a mode
other than <literal>"none"</literal> requires
you to specify a <option>passCommand</option>
or a <option>passphrase</option>.
'';
};
encryption.passCommand = mkOption {
type = with types; nullOr str;
description = ''
A command which prints the passphrase to stdout.
Mutually exclusive with <option>passphrase</option>.
'';
default = null;
example = "cat /path/to/passphrase_file";
};
encryption.passphrase = mkOption {
type = with types; nullOr str;
description = ''
The passphrase the backups are encrypted with.
Mutually exclusive with <option>passCommand</option>.
If you do not want the passphrase to be stored in the
world-readable Nix store, use <option>passCommand</option>.
'';
default = null;
};
compression = mkOption {
# "auto" is optional,
# compression mode must be given,
# compression level is optional
type = types.strMatching "none|(auto,)?(lz4|zstd|zlib|lzma)(,[[:digit:]]{1,2})?";
description = ''
Compression method to use. Refer to
<command>borg help compression</command>
for all available options.
'';
default = "lz4";
example = "auto,lzma";
};
exclude = mkOption {
type = with types; listOf str;
description = ''
Exclude paths matching any of the given patterns. See
<command>borg help patterns</command> for pattern syntax.
'';
default = [ ];
example = [
"/home/*/.cache"
"/nix"
];
};
doInit = mkOption {
type = types.bool;
description = ''
Run <command>borg init</command> if the
specified <option>repo</option> does not exist.
You should set this to <literal>false</literal>
if the repository is located on an external drive
that might not always be mounted.
'';
default = true;
};
appendFailedSuffix = mkOption {
type = types.bool;
description = ''
Append a <literal>.failed</literal> suffix
to the archive name, which is only removed if
<command>borg create</command> has a zero exit status.
'';
default = true;
};
prune.keep = mkOption {
# Specifying e.g. `prune.keep.yearly = -1`
# means there is no limit of yearly archives to keep
# The regex is for use with e.g. --keep-within 1y
type = with types; attrsOf (either int (strMatching "[[:digit:]]+[Hdwmy]"));
description = ''
Prune a repository by deleting all archives not matching any of the
specified retention options. See <command>borg help prune</command>
for the available options.
'';
default = { };
example = literalExample ''
{
within = "1d"; # Keep all archives from the last day
daily = 7;
weekly = 4;
monthly = -1; # Keep at least one archive for each month
}
'';
};
prune.prefix = mkOption {
type = types.str;
description = ''
Only consider archive names starting with this prefix for pruning.
By default, only archives created by this job are considered.
Use <literal>""</literal> to consider all archives.
'';
default = config.archiveBaseName;
defaultText = "\${archiveBaseName}";
};
environment = mkOption {
type = with types; attrsOf str;
description = ''
Environment variables passed to the backup script.
You can for example specify which SSH key to use.
'';
default = { };
example = { BORG_RSH = "ssh -i /path/to/key"; };
};
preHook = mkOption {
type = types.lines;
description = ''
Shell commands to run before the backup.
This can for example be used to mount file systems.
'';
default = "";
example = ''
# To add excluded paths at runtime
extraCreateArgs="$extraCreateArgs --exclude /some/path"
'';
};
postInit = mkOption {
type = types.lines;
description = ''
Shell commands to run after <command>borg init</command>.
'';
default = "";
};
postCreate = mkOption {
type = types.lines;
description = ''
Shell commands to run after <command>borg create</command>. The name
of the created archive is stored in <literal>$archiveName</literal>.
'';
default = "";
};
postPrune = mkOption {
type = types.lines;
description = ''
Shell commands to run after <command>borg prune</command>.
'';
default = "";
};
postHook = mkOption {
type = types.lines;
description = ''
Shell commands to run just before exit. They are executed
even if a previous command exits with a non-zero exit code.
The latter is available as <literal>$exitStatus</literal>.
'';
default = "";
};
extraInitArgs = mkOption {
type = types.str;
description = ''
Additional arguments for <command>borg init</command>.
Can also be set at runtime using <literal>$extraInitArgs</literal>.
'';
default = "";
example = "--append-only";
};
extraCreateArgs = mkOption {
type = types.str;
description = ''
Additional arguments for <command>borg create</command>.
Can also be set at runtime using <literal>$extraCreateArgs</literal>.
'';
default = "";
example = "--stats --checkpoint-interval 600";
};
extraPruneArgs = mkOption {
type = types.str;
description = ''
Additional arguments for <command>borg prune</command>.
Can also be set at runtime using <literal>$extraPruneArgs</literal>.
'';
default = "";
example = "--save-space";
};
};
}
));
};
options.services.borgbackup.repos = mkOption {
description = ''
Serve BorgBackup repositories to given public SSH keys,
restricting their access to the repository only.
Also, clients do not need to specify the absolute path when accessing the repository,
i.e. <literal>user@machine:.</literal> is enough. (Note colon and dot.)
'';
default = { };
type = types.attrsOf (types.submodule (
{ name, config, ... }: {
options = {
path = mkOption {
type = types.path;
description = ''
Where to store the backups. Note that the directory
is created automatically, with correct permissions.
'';
default = "/var/lib/borgbackup";
};
user = mkOption {
type = types.str;
description = ''
The user <command>borg serve</command> is run as.
User or group needs write permission
for the specified <option>path</option>.
'';
default = "borg";
};
group = mkOption {
type = types.str;
description = ''
The group <command>borg serve</command> is run as.
User or group needs write permission
for the specified <option>path</option>.
'';
default = "borg";
};
authorizedKeys = mkOption {
type = with types; listOf str;
description = ''
Public SSH keys that are given full write access to this repository.
You should use a different SSH key for each repository you write to, because
the specified keys are restricted to running <command>borg serve</command>
and can only access this single repository.
'';
default = [ ];
};
authorizedKeysAppendOnly = mkOption {
type = with types; listOf str;
description = ''
Public SSH keys that can only be used to append new data (archives) to the repository.
Note that archives can still be marked as deleted and are subsequently removed from disk
upon accessing the repo with full write access, e.g. when pruning.
'';
default = [ ];
};
allowSubRepos = mkOption {
type = types.bool;
description = ''
Allow clients to create repositories in subdirectories of the
specified <option>path</option>. These can be accessed using
<literal>user@machine:path/to/subrepo</literal>. Note that a
<option>quota</option> applies to repositories independently.
Therefore, if this is enabled, clients can create multiple
repositories and upload an arbitrary amount of data.
'';
default = false;
};
quota = mkOption {
# See the definition of parse_file_size() in src/borg/helpers/parseformat.py
type = with types; nullOr (strMatching "[[:digit:].]+[KMGTP]?");
description = ''
Storage quota for the repository. This quota is ensured for all
sub-repositories if <option>allowSubRepos</option> is enabled
but not for the overall storage space used.
'';
default = null;
example = "100G";
};
};
}
));
};
###### implementation
config = mkIf (with config.services.borgbackup; jobs != { } || repos != { })
(with config.services.borgbackup; {
assertions =
mapAttrsToList mkPassAssertion jobs
++ mapAttrsToList mkKeysAssertion repos;
system.activationScripts = mapAttrs' mkActivationScript jobs;
systemd.services =
# A job named "foo" is mapped to systemd.services.borgbackup-job-foo
mapAttrs' mkBackupService jobs
# A repo named "foo" is mapped to systemd.services.borgbackup-repo-foo
// mapAttrs' mkRepoService repos;
users = mkMerge (mapAttrsToList mkUsersConfig repos);
environment.systemPackages = with pkgs; [ borgbackup ];
});
}

View file

@ -1,21 +1,162 @@
import ./make-test.nix ({ pkgs, ...}: {
import ./make-test.nix ({ pkgs, ... }:
let
passphrase = "supersecret";
dataDir = "/ran:dom/data";
excludeFile = "not_this_file";
keepFile = "important_file";
keepFileData = "important_data";
localRepo = "/root/back:up";
archiveName = "my_archive";
remoteRepo = "borg@server:."; # No need to specify path
privateKey = pkgs.writeText "id_ed25519" ''
-----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
QyNTUxOQAAACBx8UB04Q6Q/fwDFjakHq904PYFzG9pU2TJ9KXpaPMcrwAAAJB+cF5HfnBe
RwAAAAtzc2gtZWQyNTUxOQAAACBx8UB04Q6Q/fwDFjakHq904PYFzG9pU2TJ9KXpaPMcrw
AAAEBN75NsJZSpt63faCuaD75Unko0JjlSDxMhYHAPJk2/xXHxQHThDpD9/AMWNqQer3Tg
9gXMb2lTZMn0pelo8xyvAAAADXJzY2h1ZXR6QGt1cnQ=
-----END OPENSSH PRIVATE KEY-----
'';
publicKey = ''
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHxQHThDpD9/AMWNqQer3Tg9gXMb2lTZMn0pelo8xyv root@client
'';
privateKeyAppendOnly = pkgs.writeText "id_ed25519" ''
-----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
QyNTUxOQAAACBacZuz1ELGQdhI7PF6dGFafCDlvh8pSEc4cHjkW0QjLwAAAJC9YTxxvWE8
cQAAAAtzc2gtZWQyNTUxOQAAACBacZuz1ELGQdhI7PF6dGFafCDlvh8pSEc4cHjkW0QjLw
AAAEAAhV7wTl5dL/lz+PF/d4PnZXuG1Id6L/mFEiGT1tZsuFpxm7PUQsZB2Ejs8Xp0YVp8
IOW+HylIRzhweORbRCMvAAAADXJzY2h1ZXR6QGt1cnQ=
-----END OPENSSH PRIVATE KEY-----
'';
publicKeyAppendOnly = ''
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFpxm7PUQsZB2Ejs8Xp0YVp8IOW+HylIRzhweORbRCMv root@client
'';
in {
name = "borgbackup";
meta = with pkgs.stdenv.lib.maintainers; {
maintainers = [ mic92 ];
meta = with pkgs.stdenv.lib; {
maintainers = with maintainers; [ dotlambda ];
};
nodes = {
machine = { config, pkgs, ... }: {
environment.systemPackages = [ pkgs.borgbackup ];
client = { config, pkgs, ... }: {
services.borgbackup.jobs = {
local = rec {
paths = dataDir;
repo = localRepo;
preHook = ''
# Don't append a timestamp
archiveName="${archiveName}"
'';
encryption = {
mode = "repokey";
inherit passphrase;
};
compression = "auto,zlib,9";
prune.keep = {
within = "1y";
yearly = 5;
};
exclude = [ "*/${excludeFile}" ];
postHook = "echo post";
startAt = [ ]; # Do not run automatically
};
remote = {
paths = dataDir;
repo = remoteRepo;
encryption.mode = "none";
startAt = [ ];
environment.BORG_RSH = "ssh -oStrictHostKeyChecking=no -i /root/id_ed25519";
};
remoteAppendOnly = {
paths = dataDir;
repo = remoteRepo;
encryption.mode = "none";
startAt = [ ];
environment.BORG_RSH = "ssh -oStrictHostKeyChecking=no -i /root/id_ed25519.appendOnly";
};
};
};
server = { config, pkgs, ... }: {
services.openssh = {
enable = true;
passwordAuthentication = false;
challengeResponseAuthentication = false;
};
services.borgbackup.repos.repo1 = {
authorizedKeys = [ publicKey ];
path = "/data/borgbackup";
};
# Second repo to make sure the authorizedKeys options are merged correctly
services.borgbackup.repos.repo2 = {
authorizedKeysAppendOnly = [ publicKeyAppendOnly ];
path = "/data/borgbackup";
quota = ".5G";
};
};
};
testScript = ''
my $borg = "BORG_PASSPHRASE=supersecret borg";
$machine->succeed("$borg init --encryption=repokey /tmp/backup");
$machine->succeed("mkdir /tmp/data/ && echo 'data' >/tmp/data/file");
$machine->succeed("$borg create --stats /tmp/backup::test /tmp/data");
$machine->succeed("$borg extract /tmp/backup::test");
$machine->succeed('c=$(cat data/file) && echo "c = $c" >&2 && [[ "$c" == "data" ]]');
startAll;
$client->fail('test -d "${remoteRepo}"');
$client->succeed("cp ${privateKey} /root/id_ed25519");
$client->succeed("chmod 0600 /root/id_ed25519");
$client->succeed("cp ${privateKeyAppendOnly} /root/id_ed25519.appendOnly");
$client->succeed("chmod 0600 /root/id_ed25519.appendOnly");
$client->succeed("mkdir -p ${dataDir}");
$client->succeed("touch ${dataDir}/${excludeFile}");
$client->succeed("echo '${keepFileData}' > ${dataDir}/${keepFile}");
subtest "local", sub {
my $borg = "BORG_PASSPHRASE='${passphrase}' borg";
$client->systemctl("start --wait borgbackup-job-local");
$client->fail("systemctl is-failed borgbackup-job-local");
# Make sure exactly one archive has been created
$client->succeed("c=\$($borg list '${localRepo}' | wc -l) && [[ \$c == '1' ]]");
# Make sure excludeFile has been excluded
$client->fail("$borg list '${localRepo}::${archiveName}' | grep -qF '${excludeFile}'");
# Make sure keepFile has the correct content
$client->succeed("$borg extract '${localRepo}::${archiveName}'");
$client->succeed('c=$(cat ${dataDir}/${keepFile}) && [[ "$c" == "${keepFileData}" ]]');
};
subtest "remote", sub {
my $borg = "BORG_RSH='ssh -oStrictHostKeyChecking=no -i /root/id_ed25519' borg";
$server->waitForUnit("sshd.service");
$client->waitForUnit("network.target");
$client->systemctl("start --wait borgbackup-job-remote");
$client->fail("systemctl is-failed borgbackup-job-remote");
# Make sure we can't access repos other than the specified one
$client->fail("$borg list borg\@server:wrong");
#TODO: Make sure that data is actually deleted
};
subtest "remoteAppendOnly", sub {
my $borg = "BORG_RSH='ssh -oStrictHostKeyChecking=no -i /root/id_ed25519.appendOnly' borg";
$server->waitForUnit("sshd.service");
$client->waitForUnit("network.target");
$client->systemctl("start --wait borgbackup-job-remoteAppendOnly");
$client->fail("systemctl is-failed borgbackup-job-remoteAppendOnly");
# Make sure we can't access repos other than the specified one
$client->fail("$borg list borg\@server:wrong");
#TODO: Make sure that data is not actually deleted
};
'';
})