diff --git a/configs/hass/default.nix b/configs/hass/default.nix index c636cce8..b198a49a 100644 --- a/configs/hass/default.nix +++ b/configs/hass/default.nix @@ -51,4 +51,7 @@ }; environment.systemPackages = [ pkgs.mosquitto ]; + + # retained messages; mosquitto.db is replaced atomically on every save + lass.backup.state.mosquitto.folders = [ config.services.mosquitto.dataDir ]; } diff --git a/configs/hass/zigbee.nix b/configs/hass/zigbee.nix index 4464e5ce..cc611bed 100644 --- a/configs/hass/zigbee.nix +++ b/configs/hass/zigbee.nix @@ -89,4 +89,11 @@ }; }; }; + + # devices, groups and the coordinator backup; database.db is JSON, not + # SQLite, so a plain copy is fine + lass.backup.state.zigbee2mqtt = { + folders = [ "/var/lib/zigbee2mqtt" ]; + exclude = [ "/var/lib/zigbee2mqtt/log" ]; + }; } diff --git a/configs/n8n.nix b/configs/n8n.nix index 4f8703f7..936c5187 100644 --- a/configs/n8n.nix +++ b/configs/n8n.nix @@ -1,4 +1,5 @@ { + pkgs, ... }: { @@ -10,4 +11,16 @@ networking.firewall.interfaces.retiolum.allowedTCPPorts = [ 5678 ]; networking.firewall.interfaces.wiregrill.allowedTCPPorts = [ 5678 ]; + + # DynamicUser state (config holds the credentials' encryption key); the + # SQLite database is dumped consistently instead of copied. + lass.backup.state.n8n = { + folders = [ "/var/lib/private/n8n" ]; + exclude = [ "/var/lib/private/n8n/.n8n/database.sqlite*" ]; + preBackup = '' + ${pkgs.sqlite}/bin/sqlite3 -cmd ".timeout 60000" /var/lib/private/n8n/.n8n/database.sqlite ".backup /var/backup/n8n.sqlite" + ''; + # WAL-mode SQLite readers need to write the -shm file + readWritePaths = [ "/var/lib/private/n8n/.n8n" ]; + }; } diff --git a/configs/services/flix/default.nix b/configs/services/flix/default.nix index cb292947..a559f218 100644 --- a/configs/services/flix/default.nix +++ b/configs/services/flix/default.nix @@ -1,4 +1,9 @@ -{ config, pkgs, ... }: +{ + config, + lib, + pkgs, + ... +}: { # sabnzbd hard-depends on cheetah3, whose PyPI dist is `ct3` while nixpkgs # sets pname="cheetah3". pythonMetadataCheckPhase then looks up metadata for @@ -465,4 +470,62 @@ systemd.tmpfiles.rules = [ "e /var/download/usenet - - - 7d" ]; + + # Configs without media, downloads, logs and image caches; the SQLite + # databases are dumped consistently instead of copied. + lass.backup.state.flix = + let + dbs = { + jellyfin = "/var/lib/jellyfin/data/*.db"; + sonarr = "${config.services.sonarr.dataDir}/sonarr.db"; + radarr = "${config.services.radarr.dataDir}/radarr.db"; + prowlarr = "/var/lib/private/prowlarr/prowlarr.db"; + jellyseerr = "/var/lib/private/jellyseerr/config/db/db.sqlite3"; + sabnzbd = "/var/lib/sabnzbd/admin/history1.db"; + }; + dirs = [ + "/var/lib/jellyfin" + config.services.sonarr.dataDir + config.services.radarr.dataDir + "/var/lib/private/prowlarr" + "/var/lib/private/jellyseerr" + "/var/lib/sabnzbd" + ]; + in + { + folders = dirs; + exclude = [ + "/var/lib/jellyfin/metadata" + "/var/lib/jellyfin/log" + "/var/lib/jellyfin/transcodes" + "${config.services.sonarr.dataDir}/MediaCover" + "${config.services.sonarr.dataDir}/logs" + "${config.services.radarr.dataDir}/MediaCover" + "${config.services.radarr.dataDir}/logs" + "/var/lib/private/prowlarr/logs" + "/var/lib/private/jellyseerr/config/cache" + "/var/lib/private/jellyseerr/config/logs" + "/var/lib/private/jellyseerr/config/db/db.sqlite3*" + "/var/lib/sabnzbd/Downloads" + "/var/lib/sabnzbd/logs" + ] + # the live databases, also the unneeded servarr logs.db + ++ map (dir: "${dir}/*.db") dirs + ++ map (dir: "${dir}/*.db-*") dirs; + preBackup = '' + rm -rf /var/backup/flix + ${lib.concatStrings ( + lib.mapAttrsToList (name: glob: '' + mkdir -p /var/backup/flix/${name} + for db in ${glob}; do + # sqlite3 would create a missing database + test -f "$db" + ${pkgs.sqlite}/bin/sqlite3 -cmd ".timeout 60000" "$db" ".backup '/var/backup/flix/${name}/$(basename "$db")'" + done + '') dbs + )} + ''; + # WAL-mode SQLite readers need to write the -shm file + readWritePaths = map dirOf (lib.attrValues dbs); + }; } diff --git a/machines/acheron/backup.nix b/machines/acheron/backup.nix new file mode 100644 index 00000000..8d776f48 --- /dev/null +++ b/machines/acheron/backup.nix @@ -0,0 +1 @@ +{ lass.backup.user = "u550643-sub4"; } diff --git a/machines/acheron/config.nix b/machines/acheron/config.nix index a10dc275..23032b25 100644 --- a/machines/acheron/config.nix +++ b/machines/acheron/config.nix @@ -7,6 +7,7 @@ ../../configs/hass ../../configs/gg23/audio/hub.nix ../../configs/gg23/internet.nix + ./backup.nix ]; services.earlyoom.enable = true; diff --git a/machines/coaxmetal/backup.nix b/machines/coaxmetal/backup.nix new file mode 100644 index 00000000..26a1de3b --- /dev/null +++ b/machines/coaxmetal/backup.nix @@ -0,0 +1 @@ +{ lass.backup.user = "u550643-sub2"; } diff --git a/machines/coaxmetal/config.nix b/machines/coaxmetal/config.nix index 2a24ecdb..f98077d1 100644 --- a/machines/coaxmetal/config.nix +++ b/machines/coaxmetal/config.nix @@ -23,6 +23,7 @@ ./strom.nix ./hermes.nix ./hermes-voice.nix + ./backup.nix ]; system.stateVersion = "24.05"; diff --git a/machines/coaxmetal/hermes.nix b/machines/coaxmetal/hermes.nix index 76de8e2e..7606effa 100644 --- a/machines/coaxmetal/hermes.nix +++ b/machines/coaxmetal/hermes.nix @@ -639,4 +639,39 @@ in EOF ''; }; + + # The home without caches, toolchains and git clones; the SQLite databases + # under .hermes are dumped consistently instead of copied. + lass.backup.state.hermes = + let + home = config.services.hermes-agent.stateDir; + in + { + folders = [ home ]; + exclude = [ + "${home}/.cache" + "${home}/.cargo" + "${home}/.gradle" + "${home}/go" + "${home}/.hermes/cache" + "${home}/workspace" + "${home}/.local/share/Trash" + "${home}/*/node_modules" + "${home}/*/target" + "${home}/.hermes/*.db" + "${home}/.hermes/*.db-wal" + "${home}/.hermes/*.db-shm" + ]; + preBackup = '' + rm -rf /var/backup/hermes + cd ${home}/.hermes + ${pkgs.findutils}/bin/find . -path ./cache -prune -o -type f -name '*.db' -printf '%P\n' | + while read -r db; do + mkdir -p "/var/backup/hermes/$(dirname "$db")" + ${pkgs.sqlite}/bin/sqlite3 -cmd ".timeout 60000" "$db" ".backup '/var/backup/hermes/$db'" + done + ''; + # WAL-mode SQLite readers need to write the -shm file + readWritePaths = [ "${home}/.hermes" ]; + }; } diff --git a/machines/neoprism/backup.nix b/machines/neoprism/backup.nix index dcfc9fab..daa363af 100644 --- a/machines/neoprism/backup.nix +++ b/machines/neoprism/backup.nix @@ -1,11 +1 @@ -{ - # Hetzner storage box; what gets backed up is declared next to each service - # (lass.backup.state). - lass.backup = { - repo = "u550643@u550643.your-storagebox.de:/./neoprism"; - sshPort = 23; - knownHosts = '' - [u550643.your-storagebox.de]:23 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs - ''; - }; -} +{ lass.backup.user = "u550643-sub1"; } diff --git a/machines/styx/backup.nix b/machines/styx/backup.nix new file mode 100644 index 00000000..eb0325ff --- /dev/null +++ b/machines/styx/backup.nix @@ -0,0 +1 @@ +{ lass.backup.user = "u550643-sub3"; } diff --git a/machines/styx/config.nix b/machines/styx/config.nix index 2b2d536b..aaac511d 100644 --- a/machines/styx/config.nix +++ b/machines/styx/config.nix @@ -14,6 +14,7 @@ ../../configs/ppp/umts-stick.nix ../../configs/gg23/audio/room.nix ../../configs/monitoring/server.nix + ./backup.nix ]; environment.systemPackages = with pkgs; [ diff --git a/machines/yellow/backup.nix b/machines/yellow/backup.nix new file mode 100644 index 00000000..b70bdedf --- /dev/null +++ b/machines/yellow/backup.nix @@ -0,0 +1 @@ +{ lass.backup.user = "u550643-sub5"; } diff --git a/machines/yellow/config.nix b/machines/yellow/config.nix index b233d2b9..8e0dcc55 100644 --- a/machines/yellow/config.nix +++ b/machines/yellow/config.nix @@ -16,6 +16,7 @@ in ../../configs/services/flix ../../configs/autoupdate.nix ../../configs/services/archiver-bot.nix + ./backup.nix ]; services.transmission.settings.peer-port = torrentport; diff --git a/modules/backup.nix b/modules/backup.nix index c5fd8822..7b1168c4 100644 --- a/modules/backup.nix +++ b/modules/backup.nix @@ -8,6 +8,7 @@ let cfg = config.lass.backup; states = lib.attrValues cfg.state; key = config.clan.core.vars.generators.borgbackup.files."borgbackup.ssh.id25519".path; + passphrase = config.clan.core.vars.generators.borgbackup-repokey.files.passphrase.path; # Runs every state's hook in its own script, keeps going when one fails # and fails at the end if any did. @@ -30,23 +31,32 @@ let in { options.lass.backup = { - repo = lib.mkOption { + user = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; - example = "user@backup.example.com:/./machine"; + example = "u550643-sub1"; description = '' - Borg repository the machine backs up to. Without it no backup job is - generated, so modules can declare their state everywhere. + Storage box (sub-)account the machine backs up to, into ./borg in its + home. Without it no backup job is generated, so modules can declare + their state everywhere. ''; }; + host = lib.mkOption { + type = lib.types.str; + default = "u550643.your-storagebox.de"; + }; + sshPort = lib.mkOption { type = lib.types.port; - default = 22; + default = 23; }; knownHosts = lib.mkOption { type = lib.types.lines; + default = '' + [u550643.your-storagebox.de]:23 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs + ''; description = "known_hosts lines pinning the host key of the repo's server."; }; @@ -103,7 +113,7 @@ in }; }; - config = lib.mkIf (cfg.repo != null) { + config = lib.mkIf (cfg.user != null) { clan.core.vars.generators.borgbackup = { files."borgbackup.ssh.id25519" = { }; files."borgbackup.ssh.id25519.pub".secret = false; @@ -116,19 +126,29 @@ in ''; }; + clan.core.vars.generators.borgbackup-repokey = { + files.passphrase = { }; + runtimeInputs = [ pkgs.openssl ]; + script = '' + openssl rand -hex 32 > "$out"/passphrase + ''; + }; + systemd.tmpfiles.rules = [ "d /var/backup 0700 root root -" ]; - # Archives are named and pruned by -; another job name - # would leave the existing archives unpruned. + # Archives are named and pruned by -. services.borgbackup.jobs.hetzner = { - inherit (cfg) repo; + repo = "${cfg.user}@${cfg.host}:./borg"; paths = lib.unique ([ "/var/backup" ] ++ lib.concatMap (state: state.folders) states); exclude = lib.unique ([ "*.pyc" ] ++ lib.concatMap (state: state.exclude) states); readWritePaths = lib.unique ( [ "/var/backup" ] ++ lib.concatMap (state: state.readWritePaths) states ); preHook = "${runHooks "preBackup"}"; - encryption.mode = "none"; + encryption = { + mode = "repokey-blake2"; + passCommand = "cat ${passphrase}"; + }; compression = "auto,zstd"; startAt = "*-*-* 03:00:00"; environment.BORG_RSH = "ssh -oPort=${toString cfg.sshPort} -oUserKnownHostsFile=${pkgs.writeText "backup-known-hosts" cfg.knownHosts} -i ${key}"; @@ -140,6 +160,12 @@ in }; }; + # spreads the machines sharing the storage box over 03:00-04:00 + systemd.timers.borgbackup-job-hetzner.timerConfig = { + RandomizedDelaySec = "1h"; + FixedRandomDelay = true; + }; + systemd.services.borgbackup-job-hetzner.serviceConfig.ExecStopPost = lib.mkIf (lib.any ( state: state.postBackup != null ) states) [ "+${runHooks "postBackup"}" ]; diff --git a/vars/per-machine/acheron/borgbackup/borgbackup.ssh.id25519.pub/value b/vars/per-machine/acheron/borgbackup/borgbackup.ssh.id25519.pub/value new file mode 100644 index 00000000..e254028d --- /dev/null +++ b/vars/per-machine/acheron/borgbackup/borgbackup.ssh.id25519.pub/value @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMh4k5nsPE5plrYU6QP3cExqJoVeeC8SiLHheWldSpkJ nixbld@coaxmetal diff --git a/vars/per-machine/coaxmetal/borgbackup/borgbackup.ssh.id25519.pub/value b/vars/per-machine/coaxmetal/borgbackup/borgbackup.ssh.id25519.pub/value new file mode 100644 index 00000000..f1b0e010 --- /dev/null +++ b/vars/per-machine/coaxmetal/borgbackup/borgbackup.ssh.id25519.pub/value @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPy26aFabIaIZyX/NQZ/KwULO0BIN/4ky4vS95oDhkFv nixbld@coaxmetal diff --git a/vars/per-machine/styx/borgbackup/borgbackup.ssh.id25519.pub/value b/vars/per-machine/styx/borgbackup/borgbackup.ssh.id25519.pub/value new file mode 100644 index 00000000..2379f54c --- /dev/null +++ b/vars/per-machine/styx/borgbackup/borgbackup.ssh.id25519.pub/value @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINn9AasDBQOhvezrzKYmtPpe5hnLrhHrkLgbk1WDK46F nixbld@coaxmetal diff --git a/vars/per-machine/yellow/borgbackup/borgbackup.ssh.id25519.pub/value b/vars/per-machine/yellow/borgbackup/borgbackup.ssh.id25519.pub/value new file mode 100644 index 00000000..2bd88ce3 --- /dev/null +++ b/vars/per-machine/yellow/borgbackup/borgbackup.ssh.id25519.pub/value @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL2aPXHjb2NFCVdSUBAWIgLvv+VZPnGCgnTKkfAeSVAe nixbld@coaxmetal