backups: encrypted per-machine storage box sub-accounts, + coaxmetal, styx, acheron, yellow

configs/hass/default.nix

@@ -51,4 +51,7 @@
   };
 
   environment.systemPackages = [ pkgs.mosquitto ];
+
+  # retained messages; mosquitto.db is replaced atomically on every save
+  lass.backup.state.mosquitto.folders = [ config.services.mosquitto.dataDir ];
 }

configs/hass/zigbee.nix

@@ -89,4 +89,11 @@
       };
     };
   };
+
+  # devices, groups and the coordinator backup; database.db is JSON, not
+  # SQLite, so a plain copy is fine
+  lass.backup.state.zigbee2mqtt = {
+    folders = [ "/var/lib/zigbee2mqtt" ];
+    exclude = [ "/var/lib/zigbee2mqtt/log" ];
+  };
 }

configs/n8n.nix

@@ -1,4 +1,5 @@
 {
+  pkgs,
   ...
 }:
 {
@@ -10,4 +11,16 @@
 
   networking.firewall.interfaces.retiolum.allowedTCPPorts = [ 5678 ];
   networking.firewall.interfaces.wiregrill.allowedTCPPorts = [ 5678 ];
+
+  # DynamicUser state (config holds the credentials' encryption key); the
+  # SQLite database is dumped consistently instead of copied.
+  lass.backup.state.n8n = {
+    folders = [ "/var/lib/private/n8n" ];
+    exclude = [ "/var/lib/private/n8n/.n8n/database.sqlite*" ];
+    preBackup = ''
+      ${pkgs.sqlite}/bin/sqlite3 -cmd ".timeout 60000" /var/lib/private/n8n/.n8n/database.sqlite ".backup /var/backup/n8n.sqlite"
+    '';
+    # WAL-mode SQLite readers need to write the -shm file
+    readWritePaths = [ "/var/lib/private/n8n/.n8n" ];
+  };
 }

configs/services/flix/default.nix

@@ -1,4 +1,9 @@
-{ config, pkgs, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
 {
   # sabnzbd hard-depends on cheetah3, whose PyPI dist is `ct3` while nixpkgs
   # sets pname="cheetah3". pythonMetadataCheckPhase then looks up metadata for
@@ -465,4 +470,62 @@
   systemd.tmpfiles.rules = [
     "e /var/download/usenet - - - 7d"
   ];
+
+  # Configs without media, downloads, logs and image caches; the SQLite
+  # databases are dumped consistently instead of copied.
+  lass.backup.state.flix =
+    let
+      dbs = {
+        jellyfin = "/var/lib/jellyfin/data/*.db";
+        sonarr = "${config.services.sonarr.dataDir}/sonarr.db";
+        radarr = "${config.services.radarr.dataDir}/radarr.db";
+        prowlarr = "/var/lib/private/prowlarr/prowlarr.db";
+        jellyseerr = "/var/lib/private/jellyseerr/config/db/db.sqlite3";
+        sabnzbd = "/var/lib/sabnzbd/admin/history1.db";
+      };
+      dirs = [
+        "/var/lib/jellyfin"
+        config.services.sonarr.dataDir
+        config.services.radarr.dataDir
+        "/var/lib/private/prowlarr"
+        "/var/lib/private/jellyseerr"
+        "/var/lib/sabnzbd"
+      ];
+    in
+    {
+      folders = dirs;
+      exclude = [
+        "/var/lib/jellyfin/metadata"
+        "/var/lib/jellyfin/log"
+        "/var/lib/jellyfin/transcodes"
+        "${config.services.sonarr.dataDir}/MediaCover"
+        "${config.services.sonarr.dataDir}/logs"
+        "${config.services.radarr.dataDir}/MediaCover"
+        "${config.services.radarr.dataDir}/logs"
+        "/var/lib/private/prowlarr/logs"
+        "/var/lib/private/jellyseerr/config/cache"
+        "/var/lib/private/jellyseerr/config/logs"
+        "/var/lib/private/jellyseerr/config/db/db.sqlite3*"
+        "/var/lib/sabnzbd/Downloads"
+        "/var/lib/sabnzbd/logs"
+      ]
+      # the live databases, also the unneeded servarr logs.db
+      ++ map (dir: "${dir}/*.db") dirs
+      ++ map (dir: "${dir}/*.db-*") dirs;
+      preBackup = ''
+        rm -rf /var/backup/flix
+        ${lib.concatStrings (
+          lib.mapAttrsToList (name: glob: ''
+            mkdir -p /var/backup/flix/${name}
+            for db in ${glob}; do
+              # sqlite3 would create a missing database
+              test -f "$db"
+              ${pkgs.sqlite}/bin/sqlite3 -cmd ".timeout 60000" "$db" ".backup '/var/backup/flix/${name}/$(basename "$db")'"
+            done
+          '') dbs
+        )}
+      '';
+      # WAL-mode SQLite readers need to write the -shm file
+      readWritePaths = map dirOf (lib.attrValues dbs);
+    };
 }

machines/acheron/backup.nix

@@ -0,0 +1 @@
+{ lass.backup.user = "u550643-sub4"; }

machines/acheron/config.nix

@@ -7,6 +7,7 @@
     ../../configs/hass
     ../../configs/gg23/audio/hub.nix
     ../../configs/gg23/internet.nix
+    ./backup.nix
   ];
 
   services.earlyoom.enable = true;

machines/coaxmetal/backup.nix

@@ -0,0 +1 @@
+{ lass.backup.user = "u550643-sub2"; }

machines/coaxmetal/config.nix

@@ -23,6 +23,7 @@
     ./strom.nix
     ./hermes.nix
     ./hermes-voice.nix
+    ./backup.nix
   ];
 
   system.stateVersion = "24.05";

machines/coaxmetal/hermes.nix

@@ -639,4 +639,39 @@ in
       EOF
     '';
   };
+
+  # The home without caches, toolchains and git clones; the SQLite databases
+  # under .hermes are dumped consistently instead of copied.
+  lass.backup.state.hermes =
+    let
+      home = config.services.hermes-agent.stateDir;
+    in
+    {
+      folders = [ home ];
+      exclude = [
+        "${home}/.cache"
+        "${home}/.cargo"
+        "${home}/.gradle"
+        "${home}/go"
+        "${home}/.hermes/cache"
+        "${home}/workspace"
+        "${home}/.local/share/Trash"
+        "${home}/*/node_modules"
+        "${home}/*/target"
+        "${home}/.hermes/*.db"
+        "${home}/.hermes/*.db-wal"
+        "${home}/.hermes/*.db-shm"
+      ];
+      preBackup = ''
+        rm -rf /var/backup/hermes
+        cd ${home}/.hermes
+        ${pkgs.findutils}/bin/find . -path ./cache -prune -o -type f -name '*.db' -printf '%P\n' |
+          while read -r db; do
+            mkdir -p "/var/backup/hermes/$(dirname "$db")"
+            ${pkgs.sqlite}/bin/sqlite3 -cmd ".timeout 60000" "$db" ".backup '/var/backup/hermes/$db'"
+          done
+      '';
+      # WAL-mode SQLite readers need to write the -shm file
+      readWritePaths = [ "${home}/.hermes" ];
+    };
 }

machines/neoprism/backup.nix

@@ -1,11 +1 @@
-{
-  # Hetzner storage box; what gets backed up is declared next to each service
-  # (lass.backup.state).
-  lass.backup = {
-    repo = "u550643@u550643.your-storagebox.de:/./neoprism";
-    sshPort = 23;
-    knownHosts = ''
-      [u550643.your-storagebox.de]:23 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs
-    '';
-  };
-}
+{ lass.backup.user = "u550643-sub1"; }

machines/styx/backup.nix

@@ -0,0 +1 @@
+{ lass.backup.user = "u550643-sub3"; }

machines/styx/config.nix

@@ -14,6 +14,7 @@
     ../../configs/ppp/umts-stick.nix
     ../../configs/gg23/audio/room.nix
     ../../configs/monitoring/server.nix
+    ./backup.nix
   ];
 
   environment.systemPackages = with pkgs; [

machines/yellow/backup.nix

@@ -0,0 +1 @@
+{ lass.backup.user = "u550643-sub5"; }

machines/yellow/config.nix

@@ -16,6 +16,7 @@ in
     ../../configs/services/flix
     ../../configs/autoupdate.nix
     ../../configs/services/archiver-bot.nix
+    ./backup.nix
   ];
 
   services.transmission.settings.peer-port = torrentport;

modules/backup.nix

@@ -8,6 +8,7 @@ let
   cfg = config.lass.backup;
   states = lib.attrValues cfg.state;
   key = config.clan.core.vars.generators.borgbackup.files."borgbackup.ssh.id25519".path;
+  passphrase = config.clan.core.vars.generators.borgbackup-repokey.files.passphrase.path;
 
   # Runs every state's hook in its own script, keeps going when one fails
   # and fails at the end if any did.
@@ -30,23 +31,32 @@ let
 in
 {
   options.lass.backup = {
-    repo = lib.mkOption {
+    user = lib.mkOption {
       type = lib.types.nullOr lib.types.str;
       default = null;
-      example = "user@backup.example.com:/./machine";
+      example = "u550643-sub1";
       description = ''
-        Borg repository the machine backs up to. Without it no backup job is
-        generated, so modules can declare their state everywhere.
+        Storage box (sub-)account the machine backs up to, into ./borg in its
+        home. Without it no backup job is generated, so modules can declare
+        their state everywhere.
       '';
     };
 
+    host = lib.mkOption {
+      type = lib.types.str;
+      default = "u550643.your-storagebox.de";
+    };
+
     sshPort = lib.mkOption {
       type = lib.types.port;
-      default = 22;
+      default = 23;
     };
 
     knownHosts = lib.mkOption {
       type = lib.types.lines;
+      default = ''
+        [u550643.your-storagebox.de]:23 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs
+      '';
       description = "known_hosts lines pinning the host key of the repo's server.";
     };
 
@@ -103,7 +113,7 @@ in
     };
   };
 
-  config = lib.mkIf (cfg.repo != null) {
+  config = lib.mkIf (cfg.user != null) {
     clan.core.vars.generators.borgbackup = {
       files."borgbackup.ssh.id25519" = { };
       files."borgbackup.ssh.id25519.pub".secret = false;
@@ -116,19 +126,29 @@ in
       '';
     };
 
+    clan.core.vars.generators.borgbackup-repokey = {
+      files.passphrase = { };
+      runtimeInputs = [ pkgs.openssl ];
+      script = ''
+        openssl rand -hex 32 > "$out"/passphrase
+      '';
+    };
+
     systemd.tmpfiles.rules = [ "d /var/backup 0700 root root -" ];
 
-    # Archives are named and pruned by <hostname>-<job>; another job name
-    # would leave the existing archives unpruned.
+    # Archives are named and pruned by <hostname>-<job>.
     services.borgbackup.jobs.hetzner = {
-      inherit (cfg) repo;
+      repo = "${cfg.user}@${cfg.host}:./borg";
       paths = lib.unique ([ "/var/backup" ] ++ lib.concatMap (state: state.folders) states);
       exclude = lib.unique ([ "*.pyc" ] ++ lib.concatMap (state: state.exclude) states);
       readWritePaths = lib.unique (
         [ "/var/backup" ] ++ lib.concatMap (state: state.readWritePaths) states
       );
       preHook = "${runHooks "preBackup"}";
-      encryption.mode = "none";
+      encryption = {
+        mode = "repokey-blake2";
+        passCommand = "cat ${passphrase}";
+      };
       compression = "auto,zstd";
       startAt = "*-*-* 03:00:00";
       environment.BORG_RSH = "ssh -oPort=${toString cfg.sshPort} -oUserKnownHostsFile=${pkgs.writeText "backup-known-hosts" cfg.knownHosts} -i ${key}";
@@ -140,6 +160,12 @@ in
       };
     };
 
+    # spreads the machines sharing the storage box over 03:00-04:00
+    systemd.timers.borgbackup-job-hetzner.timerConfig = {
+      RandomizedDelaySec = "1h";
+      FixedRandomDelay = true;
+    };
+
     systemd.services.borgbackup-job-hetzner.serviceConfig.ExecStopPost = lib.mkIf (lib.any (
       state: state.postBackup != null
     ) states) [ "+${runHooks "postBackup"}" ];

vars/per-machine/acheron/borgbackup/borgbackup.ssh.id25519.pub/value

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMh4k5nsPE5plrYU6QP3cExqJoVeeC8SiLHheWldSpkJ nixbld@coaxmetal

vars/per-machine/coaxmetal/borgbackup/borgbackup.ssh.id25519.pub/value

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPy26aFabIaIZyX/NQZ/KwULO0BIN/4ky4vS95oDhkFv nixbld@coaxmetal

vars/per-machine/styx/borgbackup/borgbackup.ssh.id25519.pub/value

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINn9AasDBQOhvezrzKYmtPpe5hnLrhHrkLgbk1WDK46F nixbld@coaxmetal

vars/per-machine/yellow/borgbackup/borgbackup.ssh.id25519.pub/value

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL2aPXHjb2NFCVdSUBAWIgLvv+VZPnGCgnTKkfAeSVAe nixbld@coaxmetal