Add zfs-auto-snapshot.sh script

2025-12-08 16:14:46 +01:00
15 changed files with 48 additions and 1012 deletions
@@ -3,7 +3,6 @@
 Some helpful bash scripts for [incus](https://linuxcontainers.org/incus)
 - incus-container-upgrade: Perform `apt dist-upgrade` command on running instances.
- incus-backup: Backup instances FS and DBs to a local directory (`-d local`) or a remote location using rsync over ssh.
+- incus-backup: Backup instances FS and DBs to a remote location using rsync over ssh.
- incus-copy: Run a differential copy of all instances to a remote incus server (push or pull) or to a local project/pool.
+- incus-copy: Run a differential copy of all running instances to a remote incus server.
 - incus-snapshot: Take a snapshot on all running instances.
 - zfs-auto-snapshot: Create and cleanup expired zfs snapshots.
@@ -30,5 +30,12 @@
    "solar": {
        "DB": ["solar"],
        "FS": ["/var/www/html/solar"]
    },
    "spot": {
        "DB": ["spot"],
        "FS": ["/var/www"]
    },
    "wedding": {
        "FS": ["/var/www/mariage"]
    }
 }
@@ -1,106 +1,55 @@
 #!/bin/bash
 #
-# Backup instances FS and DBs, driven by a JSON manifest (see
+# Backup instances FS and DBs to a remote location using rsync over ssh.
 # incus-backup.db: {"<instance>": {"DB": [...], "FS": [...]}, ...}).
 # Destination is either a remote (rsync over ssh) or, with -d local,
 # a local directory (e.g. a dedicated backup zpool mounted on /backup).
 #
 # Usage: incus-backup.sh -f <manifest> -s <storage-pool>
 #          [-d <dest-host|local>] [-u <ssh-user>] [-i <ssh-key>]
 #          [-p <ssh-port>] [-b <backup-root>]
 #
 # Layout under <backup-root> (default /backup): <hostname>/<instance>/
 #   plus <hostname>/incus-{local,global}-db.sql
-set -u
+while getopts d:u:i:p:f:s: flag
-
+do
 DEST=local
 SSH_USER=""
 SSH_KEY=""
 SSH_PORT=22
 DB_FILE=""
 STORAGE_POOL=""
 BKP_ROOT=/backup
 LOCKFILE=/run/lock/incus-backup.lock
 usage() {
    echo "Usage: $0 -f <manifest.db> -s <storage-pool> [-d <dest-host|local>] [-u <ssh-user>] [-i <ssh-key>] [-p <ssh-port>] [-b <backup-root>]" >&2
    exit 2
 }
 while getopts d:u:i:p:f:s:b: flag; do
    case "${flag}" in
        d) DEST=${OPTARG};;
-        u) SSH_USER=${OPTARG};;
+        u) USER=${OPTARG};;
        i) SSH_KEY=${OPTARG};;
        p) SSH_PORT=${OPTARG};;
        f) DB_FILE=${OPTARG};;
        s) STORAGE_POOL=${OPTARG};;
        b) BKP_ROOT=${OPTARG};;
        *) usage;;
    esac
 done
 [ -n "$DB_FILE" ] && [ -n "$STORAGE_POOL" ] || usage
 if [ "$DEST" != "local" ]; then
    [ -n "$SSH_USER" ] && [ -n "$SSH_KEY" ] || usage
 fi
 HOST=$(hostname -s)
-BKP_DIR="${BKP_ROOT}/${HOST}"
+BKP_DIR="/backup/${HOST}"
 SSH_CMD="ssh -i ${SSH_KEY} -p ${SSH_PORT}"
 CT_PREFIX="/var/lib/incus/storage-pools/${STORAGE_POOL}/containers"
 # save <dst-file>          — write stdin to a file at the destination
 # transfer <opts> <src> <dst-dir>  — rsync a path to the destination
 if [ "$DEST" = "local" ]; then
    save()     { mkdir -p "$(dirname "$1")" && cat - > "$1"; }
    transfer() { mkdir -p "$3" && /usr/bin/rsync $1 "$2" "$3"; }
 else
    SSH_CMD="ssh -i ${SSH_KEY} -p ${SSH_PORT}"
    save()     { ${SSH_CMD} "${SSH_USER}@${DEST}" "mkdir -p $(dirname "$1") && cat - > $1"; }
    transfer() { /usr/bin/rsync $1 -e "${SSH_CMD}" "$2" "${SSH_USER}@${DEST}:$3"; }
 fi
 # refuse to overlap with a previous, still-running invocation
 exec 9> "$LOCKFILE"
 if ! flock -n 9; then
    echo "another incus-backup run holds $LOCKFILE, aborting" >&2
    exit 1
 fi
 RC=0
 # Backup incus DB
-/usr/bin/incus admin sql local .dump  | save "${BKP_DIR}/incus-local-db.sql"  || RC=1
+/usr/bin/incus admin sql local .dump | ${SSH_CMD} ${USER}@${DEST} "cat - > ${BKP_DIR}/incus-local-db.sql"
-/usr/bin/incus admin sql global .dump | save "${BKP_DIR}/incus-global-db.sql" || RC=1
+/usr/bin/incus admin sql global .dump | ${SSH_CMD} ${USER}@${DEST} "cat - > ${BKP_DIR}/incus-global-db.sql"
-for CT in $(jq -r 'keys[]' "${DB_FILE}") ; do
+for CT in $(cat ${DB_FILE} | jq -r 'keys[]') ; do
    SRC_DIR="${CT_PREFIX}/${CT}"
    DST_DIR="${BKP_DIR}/${CT}"
    # Backup container info
    if [ -f "${SRC_DIR}/backup.yaml" ] ; then
-        echo "[$(date '+%F %T')] Starting $CT backup.yaml"
+        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting $CT backup.yaml"
-        transfer "-a --del" "${SRC_DIR}/backup.yaml" "${DST_DIR}/" || RC=1
+        /usr/bin/rsync -a --del -e "${SSH_CMD}" ${SRC_DIR}/backup.yaml ${USER}@${DEST}:${DST_DIR}/
    fi
    # Backup Mysql dumps
-    for DB in $(jq -r ".${CT} | select(.DB != null) | .DB[]" "${DB_FILE}") ; do
+    for DB in $(cat $DB_FILE | jq -r ".${CT} | select(.DB != null) | .DB[]") ; do
-        echo "[$(date '+%F %T')] Starting $CT $DB database backup"
+        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting $CT $DB database backup"
-        /usr/bin/incus exec "$CT" -- mariadb-dump --single-transaction --databases "${DB}" \
+        /usr/bin/incus exec -n $CT -- mariadb-dump --single-transaction --databases ${DB} | ${SSH_CMD} ${USER}@${DEST} "mkdir -p ${DST_DIR} ; cat - > ${DST_DIR}/mysql-${DB}.sql"
            | save "${DST_DIR}/mysql-${DB}.sql" || RC=1
    done
-    # Backup container rootfs paths
+    # Backup container rootfs paths 
-    for FS in $(jq -r ".${CT} | select(.FS != null) | .FS[]" "${DB_FILE}") ; do
+    for FS in $(cat $DB_FILE | jq -r ".${CT} | select(.FS != null) | .FS[]") ; do
        # Skip missing rootfs dir
-        if [ ! -d "${SRC_DIR}/rootfs" ] ; then
+        if [ ! -d ${SRC_DIR}/rootfs ] ; then
-            echo "[$(date '+%F %T')] WARNING : Missing rootfs for container ${CT}, skipping FS ${FS} ..."
+            echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARNING : Missing rootfs for container ${CT}, skipping FS ${FS} ..."
            continue
        fi
-        # "/." used by rsync to limit the amount of path information that is sent as implied directories
+	    # "/." used by rsync to limit the amount of path information that is sent as implied directories
-        echo "[$(date '+%F %T')] Starting $CT $FS filesystem backup"
+        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting $CT $FS filesystem backup"
-        transfer "-aR --del" "${SRC_DIR}/rootfs/.${FS}" "${DST_DIR}/" || RC=1
+	    /usr/bin/rsync -aR --del -e "${SSH_CMD}" ${SRC_DIR}/rootfs/.${FS} ${USER}@${DEST}:${DST_DIR}/
    done
 done
 exit $RC
@@ -1,71 +1,15 @@
 #!/bin/bash
 #
 # apt dist-upgrade of all RUNNING containers (containers only — VMs are
 # excluded, they may not run an agent or apt at all). Containers without
 # apt are skipped. Exits non-zero if any upgrade failed.
 #
 # Also refreshes `user.os` / `user.os-checked` on each container, so
 #   incus list -c n,config:user.os,config:user.os-checked
 # always shows the OS the container actually runs. (`image.description`
 # is deliberately left alone: it records what the instance was created
 # from — 2019 images for most of this fleet — which is history worth
 # keeping, not an inventory.)
 #
 # Usage: incus-container-upgrade.sh [-o]     # -o: only refresh user.os,
 #                                            #     skip the upgrades
 set -uo pipefail
 INCUS=/usr/bin/incus
 LOG=/var/log/incus-container-upgrade.log
 LOCKFILE=/run/lock/incus-container-upgrade.lock
 OSONLY=0
 while getopts o flag; do
    case "${flag}" in
        o) OSONLY=1;;
        *) echo "Usage: $0 [-o]" >&2; exit 2;;
    esac
 done
 # refuse to overlap with a previous, still-running invocation
 exec 9> "$LOCKFILE"
 if ! flock -n 9; then
    echo "another incus-container-upgrade run holds $LOCKFILE, aborting" >&2
    exit 1
 fi
 distUpgrade() {
-    local CT=$1
+    CT=$1
-    echo -e "\n*** [$(date '+%F %T')] - Dist-upgrading $CT container ***\n"
+    DATE=$(date '+%Y-%m-%d %H:%M:%S')
-    if ! $INCUS exec "$CT" -- sh -c 'command -v apt-get >/dev/null'; then
+    echo -e "\n*** [$DATE] - Dist-upgrading $CT container ***\n"
-        echo "$CT: no apt-get in container, skipping"
+    incus exec $CT --env DEBIAN_FRONTEND=noninteractive -- apt -qq update
-        return 0
+    incus exec $CT --env DEBIAN_FRONTEND=noninteractive -- apt -qq -y dist-upgrade
    fi
    $INCUS exec "$CT" --env DEBIAN_FRONTEND=noninteractive -- apt-get -qq update &&
    $INCUS exec "$CT" -- sh -c 'apt list --upgradable 2>/dev/null' &&
    $INCUS exec "$CT" --env DEBIAN_FRONTEND=noninteractive -- apt-get -qq -y dist-upgrade
 }
-# record the OS the container actually runs; failures here never fail
+for CT in $(incus ls -c n -f compact,noheader status=RUNNING) ; do
-# the upgrade (it is metadata, not the job)
+    distUpgrade "$CT" | tee -a /var/log/incus-container-upgrade.log
 recordOS() {
    local CT=$1 OS
    OS=$($INCUS exec "$CT" -- sh -c '. /etc/os-release 2>/dev/null && echo "$PRETTY_NAME"' 2>/dev/null)
    [ -n "$OS" ] || return 0
    $INCUS config set "$CT" user.os="$OS" user.os-checked="$(date +%F)" 2>/dev/null \
        || echo "$CT: could not record user.os" >&2
 }
 RC=0
 for CT in $($INCUS list -c n -f csv status=RUNNING type=container); do
    if [ "$OSONLY" -eq 0 ]; then
        if ! distUpgrade "$CT" 2>&1 | tee -a "$LOG"; then
            echo "[$(date '+%F %T')] FAILED: $CT" | tee -a "$LOG" >&2
            RC=1
        fi
    fi
    recordOS "$CT"
 done
 exit $RC
@@ -1,86 +1,23 @@
 #!/bin/bash
 #
 # Incremental replication of ALL Incus instances (incus copy --refresh;
 # with ZFS on both ends only deltas travel, so stopped instances cost
 # nothing after their first copy).
 # https://linuxcontainers.org/incus/docs/main/howto/move_instances/
 #
 # Three shapes, per where it runs and where the replicas land:
 #   incus-copy.sh -d <dest-remote> [-m pull|push|relay] [-s <dest-pool>]
 #       push all local instances to a remote (runs on the source host)
 #   incus-copy.sh -r <src-remote> [-m pull|push|relay] [-s <dest-pool>]
 #       pull all of a remote's instances to the local host (runs on the
 #       backup host; the source only needs its API reachable)
 #   incus-copy.sh -p <target-project> -s <dest-pool>
 #       copy all local instances into another local project + pool
 #       (on-host backup onto a different disk)
 #
 # Snapshot history/retention is handled on the source instances
 # (snapshots.schedule / snapshots.expiry); --refresh-exclude-older keeps
 # refreshes from dragging expired snapshots around.
-set -u
+#set -x
-INCUS=/usr/bin/incus
+while getopts d:m: flag
-LOCKFILE=/run/lock/incus-copy.lock
+do
 SRC=""
 DEST=""
 MODE=pull
 POOL=""
 PROJECT=""
 usage() {
    echo "Usage: $0 [-d <dest-remote> | -r <src-remote>] [-m pull|push|relay] [-s <dest-storage-pool>] [-p <dest-project>]" >&2
    exit 2
 }
 while getopts d:r:m:s:p: flag; do
    case "${flag}" in
        d) DEST=${OPTARG};;
        r) SRC=${OPTARG};;
        m) MODE=${OPTARG};;
        s) POOL=${OPTARG};;
        p) PROJECT=${OPTARG};;
        *) usage;;
    esac
 done
 # need somewhere for replicas to go, and one side must stay local
 [ -n "$DEST" ] || [ -n "$SRC" ] || [ -n "$PROJECT" ] || usage
 [ -n "$DEST" ] && [ -n "$SRC" ] && usage
 case "$MODE" in pull|push|relay) ;; *) usage;; esac
 # refuse to overlap with a previous, still-running invocation
 exec 9> "$LOCKFILE"
 if ! flock -n 9; then
    echo "another incus-copy run holds $LOCKFILE, aborting" >&2
    exit 1
 fi
-INSTANCES=$($INCUS list ${SRC:+"${SRC}:"} -c n -f csv) || {
+for CT in $(/usr/bin/incus list -c n -f compact,noheader status=RUNNING) ; do
-    echo "cannot list instances${SRC:+ on ${SRC}:}, aborting" >&2
+    DATE=$(date '+%Y-%m-%d %H:%M:%S')
-    exit 1
+    CMD="/usr/bin/incus copy $CT ${DEST}:$CT --refresh --refresh-exclude-older --mode ${MODE}"
-}
+    echo "[${DATE}] $CMD"
-
+    $CMD 2>&1 > /dev/null
 RC=0
 for CT in $INSTANCES; do
    echo "[$(date '+%F %T')] copy ${SRC:+$SRC:}$CT -> ${DEST:+$DEST:}$CT${PROJECT:+ (project $PROJECT)}"
    # --mode only applies to remote transfers; --quiet suppresses the
    # \r-progress meter that turns log files into mangled one-liners
    if $INCUS copy --quiet "${SRC:+$SRC:}$CT" "${DEST:+$DEST:}$CT" \
        --refresh --refresh-exclude-older \
        ${SRC:+--mode "$MODE"} ${DEST:+--mode "$MODE"} \
        ${POOL:+--storage "$POOL"} \
        ${PROJECT:+--target-project "$PROJECT"}; then
        # the replica inherits boot.autostart from the source and would
        # start itself on the backup host after a reboot (or, for the
        # local-project shape, fight the live instance for its static
        # IP on the same bridge) — neutralize it
        $INCUS config set ${PROJECT:+--project "$PROJECT"} \
            "${DEST:+$DEST:}$CT" boot.autostart=false || RC=1
    else
        echo "[$(date '+%F %T')] FAILED: $CT" >&2
        RC=1
    fi
 done
 exit $RC
@@ -1,180 +0,0 @@
 #!/bin/bash
 #
 # Nightly plakar backup driver. Three phases:
 #   1. dumps — incus's own DBs, plus application-consistent database
 #      dumps taken with `incus exec` (container-local auth: no DB
 #      users, no network exposure). Auto-discovered in every RUNNING
 #      container — no manifest to go stale: native MariaDB
 #      (mariadb-dump, all non-system DBs), native PostgreSQL (pg_dump
 #      per DB + pg_dumpall --globals-only, peer auth as postgres),
 #      and PostgreSQL inside docker containers (image name matching
 #      "postgres", user from $POSTGRES_USER). Dumps are staged plain
 #      (not gzipped, so plakar's chunking dedups near-identical
 #      consecutive dumps) in the dump dir, then backed up as one
 #      snapshot.
 #   2. sources — every fs source listed in the sources file (one
 #      `plakar source add` name per line, #-comments allowed).
 #   3. retention — prune + maintenance on the kloset.
 # The kloset target is whatever `plakar at` accepts — normally the S3
 # store (`-k @s3`, direct-to-S3, no local staging); a path
 # (e.g. -k /backup/plakar-test) only for testing. -S optionally
 # replicates the kloset to a second store afterwards (`plakar sync`);
 # unused in the current single-store setup.
 #
 # Usage: plakar-backup.sh -k <kloset> [-f <sources-file>]
 #          [-r <retention-days>] [-S <sync-store>] [-d <dump-dir>]
 #
 # The kloset passphrase comes from $PLAKAR_PASSPHRASE or, failing
 # that, /root/.plakar-passphrase (mode 600).
 set -u
 KLOSET=""
 SRC_FILE=/root/scripts/plakar-sources
 # optional DB exclude list, one dump path per line as printed by the
 # "dump" log lines (<ct>/<db> or <ct>/<docker-name>/<db>); every skip
 # is logged loudly so the list can't rot silently like a manifest
 EXCLUDE_FILE=/root/scripts/plakar-db-exclude
 RETENTION=30
 SYNC_STORE=""
 DUMP_DIR=/backup/plakar-dumps
 LOCKFILE=/run/lock/plakar-backup.lock
 PASSFILE=/root/.plakar-passphrase
 usage() {
    echo "Usage: $0 -k <kloset> [-f <sources-file>] [-r <retention-days>] [-S <sync-store>] [-d <dump-dir>]" >&2
    exit 2
 }
 while getopts k:f:r:S:d: flag; do
    case "${flag}" in
        k) KLOSET=${OPTARG};;
        f) SRC_FILE=${OPTARG};;
        r) RETENTION=${OPTARG};;
        S) SYNC_STORE=${OPTARG};;
        d) DUMP_DIR=${OPTARG};;
        *) usage;;
    esac
 done
 [ -n "$KLOSET" ] || usage
 [ -r "$SRC_FILE" ] || { echo "sources file $SRC_FILE not readable" >&2; exit 2; }
 exec 9>"$LOCKFILE"
 if ! flock -n 9; then
    echo "another plakar-backup run holds $LOCKFILE, aborting" >&2
    exit 1
 fi
 log() { echo "[$(date '+%F %T')] $*"; }
 excluded() {
    [ -r "$EXCLUDE_FILE" ] && grep -qx "$1" "$EXCLUDE_FILE" \
        && log "SKIP $1 (listed in $EXCLUDE_FILE)"
 }
 if [ -z "${PLAKAR_PASSPHRASE:-}" ]; then
    [ -r "$PASSFILE" ] || { echo "no PLAKAR_PASSPHRASE in env and $PASSFILE not readable" >&2; exit 1; }
    PLAKAR_PASSPHRASE=$(cat "$PASSFILE")
 fi
 export PLAKAR_PASSPHRASE
 rc=0
 # incus's own state (instance configs, profiles, devices) — needed to
 # rebuild instances, not covered by any file/DB source
 mkdir -p "$DUMP_DIR/incus"
 incus admin sql global .dump > "$DUMP_DIR/incus/incus-global-db.sql" || rc=1
 incus admin sql local .dump > "$DUMP_DIR/incus/incus-local-db.sql" || rc=1
 # Database dumps, auto-discovered per running container: native
 # MariaDB, native PostgreSQL, and PostgreSQL inside docker containers
 # (matched on the docker image name containing "postgres").
 PG_LIST="SELECT datname FROM pg_database WHERE NOT datistemplate AND datname <> 'postgres'"
 for ct in $(incus list status=running -c n -f csv); do
    # native MariaDB/MySQL (unix-socket root auth) — binaries may carry
    # either naming (e.g. mail: hand-installed mariadb-dump + mysql client)
    mdump=$(incus exec "$ct" -- sh -c 'command -v mariadb-dump || command -v mysqldump' 2>/dev/null)
    mclient=$(incus exec "$ct" -- sh -c 'command -v mariadb || command -v mysql' 2>/dev/null)
    if [ -n "$mdump" ] && [ -n "$mclient" ]; then
        dbs=$(incus exec "$ct" -- "$mclient" -N -B -e 'SHOW DATABASES') \
            || { echo "listing mariadb databases on $ct failed" >&2; rc=1; dbs=""; }
        for db in $(printf '%s\n' "$dbs" \
                    | grep -Ev '^(information_schema|performance_schema|mysql|sys)$'); do
            excluded "$ct/$db" && continue
            log "dump $ct/$db (mariadb)"
            mkdir -p "$DUMP_DIR/mariadb/$ct"
            incus exec "$ct" -- "$mdump" --single-transaction --events --routines --triggers \
                --databases "$db" > "$DUMP_DIR/mariadb/$ct/$db.sql" \
                || { echo "dump $ct/$db failed" >&2; rc=1; }
        done
        # users + grants (the mariadb equivalent of pg_dumpall
        # --globals-only): replayable SHOW GRANTS statements, portable
        # across versions unlike a raw mysql-schema dump
        log "dump $ct/grants (mariadb)"
        incus exec "$ct" -- sh -c "$mclient -NBe \"SELECT CONCAT('SHOW GRANTS FOR ', QUOTE(user), '@', QUOTE(host), ';') FROM mysql.user\" | $mclient -NB | sed 's/\$/;/'" \
            > "$DUMP_DIR/mariadb/$ct/grants.sql" \
            || { echo "grants dump on $ct failed" >&2; rc=1; }
    elif [ -n "$mdump$mclient" ]; then
        echo "$ct has only one of dump/client mariadb binaries, skipping" >&2; rc=1
    fi
    # native PostgreSQL (peer auth as the postgres user)
    if incus exec "$ct" -- sh -c 'command -v pg_dump' >/dev/null 2>&1; then
        mkdir -p "$DUMP_DIR/postgres/$ct"
        incus exec "$ct" -- su -s /bin/sh postgres -c "pg_dumpall --globals-only" \
            > "$DUMP_DIR/postgres/$ct/globals.sql" || rc=1
        dbs=$(incus exec "$ct" -- su -s /bin/sh postgres -c "psql -AtX -c \"$PG_LIST\"") \
            || { echo "listing postgres databases on $ct failed" >&2; rc=1; dbs=""; }
        for db in $dbs; do
            excluded "$ct/$db" && continue
            log "dump $ct/$db (postgres)"
            incus exec "$ct" -- su -s /bin/sh postgres -c "pg_dump --clean --if-exists $db" \
                > "$DUMP_DIR/postgres/$ct/$db.sql" \
                || { echo "dump $ct/$db failed" >&2; rc=1; }
        done
    fi
    # PostgreSQL inside docker (e.g. outline, login)
    incus exec "$ct" -- sh -c 'command -v docker' >/dev/null 2>&1 || continue
    for dc in $(incus exec "$ct" -- docker ps --format '{{.Names}} {{.Image}}' 2>/dev/null \
                | awk 'tolower($2) ~ /postgres/ {print $1}'); do
        pguser=$(incus exec "$ct" -- docker exec "$dc" sh -c 'echo "${POSTGRES_USER:-postgres}"') \
            || { echo "reading POSTGRES_USER on $ct/$dc failed" >&2; rc=1; continue; }
        mkdir -p "$DUMP_DIR/postgres/$ct/$dc"
        incus exec "$ct" -- docker exec "$dc" pg_dumpall -U "$pguser" --globals-only \
            > "$DUMP_DIR/postgres/$ct/$dc/globals.sql" || rc=1
        dbs=$(incus exec "$ct" -- docker exec "$dc" psql -U "$pguser" -AtX -c "$PG_LIST") \
            || { echo "listing postgres databases on $ct/$dc failed" >&2; rc=1; dbs=""; }
        for db in $dbs; do
            excluded "$ct/$dc/$db" && continue
            log "dump $ct/$dc/$db (postgres)"
            incus exec "$ct" -- docker exec "$dc" pg_dump -U "$pguser" --clean --if-exists "$db" \
                > "$DUMP_DIR/postgres/$ct/$dc/$db.sql" \
                || { echo "dump $ct/$dc/$db failed" >&2; rc=1; }
        done
    done
 done
 log "backup dumps ($DUMP_DIR)"
 plakar -quiet at "$KLOSET" backup -tag dumps "$DUMP_DIR" \
    || { echo "backup of $DUMP_DIR failed" >&2; rc=1; }
 for src in $(grep -Ev '^[[:space:]]*(#|$)' "$SRC_FILE"); do
    log "backup @$src"
    plakar -quiet at "$KLOSET" backup -tag "$src" "@$src" \
        || { echo "backup @$src failed" >&2; rc=1; }
 done
 log "prune: keep the last $RETENTION days of snapshots"
 plakar at "$KLOSET" prune -days "$RETENTION" -apply || rc=1
 plakar at "$KLOSET" maintenance || rc=1
 if [ -n "$SYNC_STORE" ]; then
    log "sync to @$SYNC_STORE"
    plakar at "$KLOSET" sync to "@$SYNC_STORE" \
        || { echo "sync to @$SYNC_STORE failed" >&2; rc=1; }
 fi
 log "done (rc=$rc)"
 exit $rc
@@ -1,76 +0,0 @@
 #!/bin/bash
 #
 # Nightly plakar backup of the incus instances (leg 4): backs up the
 # configured incus source (the plakar incus integration walking the
 # quiesced replicas of the `backup` project) into the target kloset,
 # then applies retention. Runs CHAINED after incus-copy.sh in the same
 # cron entry — the snapshot is only as fresh as the last completed
 # local replica refresh:
 #
 #   0 1 * * * incus-copy.sh -p backup -s backup >> /var/log/incus-copy.log 2>&1 ; plakar-incus-backup.sh >> /var/log/plakar-incus.log 2>&1
 #
 # Usage: plakar-incus-backup.sh [-k <kloset>] [-s <source>] [-r <retention-days>]
 #
 # The kloset passphrase comes from $PLAKAR_PASSPHRASE or, failing
 # that, /root/.plakar-passphrase (mode 600).
 set -u
 KLOSET=@s3incus
 SOURCE=ks4-incus
 RETENTION=30
 LOCKFILE=/run/lock/plakar-incus-backup.lock
 PASSFILE=/root/.plakar-passphrase
 usage() {
    echo "Usage: $0 [-k <kloset>] [-s <source>] [-r <retention-days>]" >&2
    exit 2
 }
 while getopts k:s:r: flag; do
    case "${flag}" in
        k) KLOSET=${OPTARG};;
        s) SOURCE=${OPTARG};;
        r) RETENTION=${OPTARG};;
        *) usage;;
    esac
 done
 exec 9>"$LOCKFILE"
 if ! flock -n 9; then
    echo "another plakar-incus-backup run holds $LOCKFILE, aborting" >&2
    exit 1
 fi
 log() { echo "[$(date '+%F %T')] $*"; }
 if [ -z "${PLAKAR_PASSPHRASE:-}" ]; then
    [ -r "$PASSFILE" ] || { echo "no PLAKAR_PASSPHRASE in env and $PASSFILE not readable" >&2; exit 1; }
    PLAKAR_PASSPHRASE=$(cat "$PASSFILE")
 fi
 export PLAKAR_PASSPHRASE
 rc=0
 # The plakar pkg backend has been observed to silently expel an
 # installed plugin (its reload path deletes the ptar when the cache
 # re-extraction fails — seen 2026-08-24). Self-heal from the kept
 # ptar instead of failing the nightly run with
 # "unsupported importer protocol".
 if ! plakar pkg list 2>/dev/null | grep -q "^incus@"; then
    PTAR=$(ls -t /root/incus-plugin/incus_v*.ptar 2>/dev/null | head -1)
    echo "incus plugin missing — reinstalling ${PTAR:-<no ptar found>}" >&2
    [ -n "$PTAR" ] && plakar pkg add "$PTAR" \
        || { echo "incus plugin reinstall failed" >&2; exit 1; }
 fi
 log "backup @$SOURCE -> $KLOSET"
 plakar -quiet at "$KLOSET" backup -tag "$SOURCE" "@$SOURCE" \
    || { echo "backup @$SOURCE failed" >&2; rc=1; }
 log "prune: keep the last $RETENTION days of snapshots"
 plakar at "$KLOSET" prune -days "$RETENTION" -apply || rc=1
 plakar at "$KLOSET" maintenance || rc=1
 log "done (rc=$rc)"
 exit $rc
@@ -1,170 +0,0 @@
 #!/bin/bash
 #
 # Nightly restic data backup (repo: restic-data). Three phases:
 #   1. dumps — incus's own DBs, plus application-consistent database
 #      dumps taken with `incus exec` (container-local auth: no DB
 #      users, no network exposure). Auto-discovered in every RUNNING
 #      container — no manifest to go stale: native MariaDB/MySQL
 #      (either binary naming), native PostgreSQL (pg_dump per DB +
 #      pg_dumpall --globals-only), and PostgreSQL inside docker
 #      containers (image name matching "postgres"). Dumps staged
 #      plain (not gzipped — CDC dedup needs uncompressed input).
 #   2. one `restic backup` invocation: the dump dir + every path in
 #      the paths file (one index load, one snapshot per night);
 #      exclude patterns applied globally.
 #   3. retention — `restic forget --group-by host` (seed-era
 #      snapshots have different path sets and must age in one group).
 #      Prune/check live in restic-maintenance.sh (weekly).
 #
 # Usage: restic-backup.sh [-r <repo>] [-f <paths-file>] [-d <dump-dir>]
 #          [-s <stage>]   # stage: dumps|backup|all (default all);
 #                         # -s backup skips re-dumping (seeding aid)
 #
 # Env from /root/.restic-env (AWS creds, RESTIC_PASSWORD_FILE,
 # RESTIC_CACHE_DIR).
 set -u
 REPO=s3:s3.sbg.io.cloud.ovh.net/restic-data
 PATHS_FILE=/root/scripts/restic-paths
 EXCLUDE_FILE=/root/scripts/restic-exclude
 DB_EXCLUDE_FILE=/root/scripts/db-exclude
 # legacy name from the plakar era — keep working until the file is renamed
 [ -r "$DB_EXCLUDE_FILE" ] || [ ! -r /root/scripts/plakar-db-exclude ] || {
    DB_EXCLUDE_FILE=/root/scripts/plakar-db-exclude
    echo "note: using legacy $DB_EXCLUDE_FILE — rename it to /root/scripts/db-exclude" >&2
 }
 DUMP_DIR=/backup/dumps
 LOCKFILE=/run/lock/restic-backup.lock
 ENVFILE=/root/.restic-env
 RESTIC=/usr/local/bin/restic   # cron PATH lacks /usr/local/bin
 STAGE=all
 usage() {
    echo "Usage: $0 [-r <repo>] [-f <paths-file>] [-d <dump-dir>] [-s dumps|backup|all]" >&2
    exit 2
 }
 while getopts r:f:d:s: flag; do
    case "${flag}" in
        r) REPO=${OPTARG};;
        f) PATHS_FILE=${OPTARG};;
        d) DUMP_DIR=${OPTARG};;
        s) STAGE=${OPTARG};;
        *) usage;;
    esac
 done
 [ -r "$PATHS_FILE" ] || { echo "paths file $PATHS_FILE not readable" >&2; exit 2; }
 [ -r "$ENVFILE" ] || { echo "env file $ENVFILE not readable" >&2; exit 2; }
 . "$ENVFILE"
 exec 9>"$LOCKFILE"
 if ! flock -n 9; then
    echo "another restic-backup run holds $LOCKFILE, aborting" >&2
    exit 1
 fi
 log() { echo "[$(date '+%F %T')] $*"; }
 excluded() {
    [ -r "$DB_EXCLUDE_FILE" ] && grep -qx "$1" "$DB_EXCLUDE_FILE" \
        && log "SKIP $1 (listed in $DB_EXCLUDE_FILE)"
 }
 rc=0
 if [ "$STAGE" = all ] || [ "$STAGE" = dumps ]; then
 # incus's own state (instance configs, profiles, devices)
 mkdir -p "$DUMP_DIR/incus"
 incus admin sql global .dump > "$DUMP_DIR/incus/incus-global-db.sql" || rc=1
 incus admin sql local .dump > "$DUMP_DIR/incus/incus-local-db.sql" || rc=1
 # Database dumps, auto-discovered per running container
 PG_LIST="SELECT datname FROM pg_database WHERE NOT datistemplate AND datname <> 'postgres'"
 for ct in $(incus list status=running -c n -f csv); do
    # native MariaDB/MySQL (unix-socket root auth), either binary naming
    mdump=$(incus exec "$ct" -- sh -c 'command -v mariadb-dump || command -v mysqldump' 2>/dev/null)
    mclient=$(incus exec "$ct" -- sh -c 'command -v mariadb || command -v mysql' 2>/dev/null)
    if [ -n "$mdump" ] && [ -n "$mclient" ]; then
        dbs=$(incus exec "$ct" -- "$mclient" -N -B -e 'SHOW DATABASES') \
            || { echo "listing mariadb databases on $ct failed" >&2; rc=1; dbs=""; }
        for db in $(printf '%s\n' "$dbs" \
                    | grep -Ev '^(information_schema|performance_schema|mysql|sys)$'); do
            excluded "$ct/$db" && continue
            log "dump $ct/$db (mariadb)"
            mkdir -p "$DUMP_DIR/mariadb/$ct"
            incus exec "$ct" -- "$mdump" --single-transaction --events --routines --triggers \
                --databases "$db" > "$DUMP_DIR/mariadb/$ct/$db.sql" \
                || { echo "dump $ct/$db failed" >&2; rc=1; }
        done
        # users + grants: replayable SHOW GRANTS statements
        log "dump $ct/grants (mariadb)"
        incus exec "$ct" -- sh -c "$mclient -NBe \"SELECT CONCAT('SHOW GRANTS FOR ', QUOTE(user), '@', QUOTE(host), ';') FROM mysql.user\" | $mclient -NB | sed 's/\$/;/'" \
            > "$DUMP_DIR/mariadb/$ct/grants.sql" \
            || { echo "grants dump on $ct failed" >&2; rc=1; }
    elif [ -n "$mdump$mclient" ]; then
        echo "$ct has only one of dump/client mariadb binaries, skipping" >&2; rc=1
    fi
    # native PostgreSQL (peer auth as the postgres user)
    if incus exec "$ct" -- sh -c 'command -v pg_dump' >/dev/null 2>&1; then
        mkdir -p "$DUMP_DIR/postgres/$ct"
        incus exec "$ct" -- su -s /bin/sh postgres -c "pg_dumpall --globals-only" \
            > "$DUMP_DIR/postgres/$ct/globals.sql" || rc=1
        dbs=$(incus exec "$ct" -- su -s /bin/sh postgres -c "psql -AtX -c \"$PG_LIST\"") \
            || { echo "listing postgres databases on $ct failed" >&2; rc=1; dbs=""; }
        for db in $dbs; do
            excluded "$ct/$db" && continue
            log "dump $ct/$db (postgres)"
            incus exec "$ct" -- su -s /bin/sh postgres -c "pg_dump --clean --if-exists $db" \
                > "$DUMP_DIR/postgres/$ct/$db.sql" \
                || { echo "dump $ct/$db failed" >&2; rc=1; }
        done
    fi
    # PostgreSQL inside docker (e.g. outline, login)
    incus exec "$ct" -- sh -c 'command -v docker' >/dev/null 2>&1 || continue
    for dc in $(incus exec "$ct" -- docker ps --format '{{.Names}} {{.Image}}' 2>/dev/null \
                | awk 'tolower($2) ~ /postgres/ {print $1}'); do
        pguser=$(incus exec "$ct" -- docker exec "$dc" sh -c 'echo "${POSTGRES_USER:-postgres}"') \
            || { echo "reading POSTGRES_USER on $ct/$dc failed" >&2; rc=1; continue; }
        mkdir -p "$DUMP_DIR/postgres/$ct/$dc"
        incus exec "$ct" -- docker exec "$dc" pg_dumpall -U "$pguser" --globals-only \
            > "$DUMP_DIR/postgres/$ct/$dc/globals.sql" || rc=1
        dbs=$(incus exec "$ct" -- docker exec "$dc" psql -U "$pguser" -AtX -c "$PG_LIST") \
            || { echo "listing postgres databases on $ct/$dc failed" >&2; rc=1; dbs=""; }
        for db in $dbs; do
            excluded "$ct/$dc/$db" && continue
            log "dump $ct/$dc/$db (postgres)"
            incus exec "$ct" -- docker exec "$dc" pg_dump -U "$pguser" --clean --if-exists "$db" \
                > "$DUMP_DIR/postgres/$ct/$dc/$db.sql" \
                || { echo "dump $ct/$dc/$db failed" >&2; rc=1; }
        done
    done
 done
 fi  # stage dumps
 if [ "$STAGE" = all ] || [ "$STAGE" = backup ]; then
 # clear locks left by a killed run (safe: only stale ones are removed);
 # otherwise a single interrupted backup blocks forget/prune for ever
 $RESTIC -r "$REPO" unlock >/dev/null 2>&1 || true
 log "restic backup -> $REPO"
 $RESTIC -r "$REPO" backup \
    --pack-size 64 --read-concurrency 8 -o s3.connections=8 \
    --exclude-file "$EXCLUDE_FILE" \
    --files-from-verbatim "$PATHS_FILE" "$DUMP_DIR" \
    || { echo "restic backup failed" >&2; rc=1; }
 log "forget: keep 14d/8w/6m"
 $RESTIC -r "$REPO" forget --group-by host \
    --keep-daily 14 --keep-weekly 8 --keep-monthly 6 || rc=1
 fi  # stage backup
 log "done (rc=$rc)"
 exit $rc
@@ -1,2 +0,0 @@
 /var/lib/incus/storage-pools/data/containers/nextcloud/rootfs/nextcloud/data/appdata_*/preview
 /var/lib/incus/storage-pools/data/containers/nextcloud/rootfs/nextcloud/data/appdata_*/dav-photocache
@@ -1,121 +0,0 @@
 #!/bin/bash
 #
 # Nightly restic backup of the incus instances (repo: restic-incus).
 # For every instance of the `backup` project (quiesced replicas —
 # stopped, refreshed by the 01:00 incus-copy), mount its filesystem
 # with `incus file mount` (FUSE over the per-instance sftp API, works
 # on stopped containers, needs sshfs) and back it up per-file,
 # together with its expanded config.
 #
 # ⚠️ Each instance gets its OWN mountpoint (/run/restic-incus/<name>):
 # restic selects a snapshot's parent by host+path, so a shared
 # mountpoint would parent every snapshot on the previous *other*
 # instance and force nightly full re-reads.
 #
 # Runs CHAINED after incus-copy.sh in the same cron entry — the
 # snapshot is only as fresh as the last completed replica refresh:
 #   0 1 * * * incus-copy.sh -p backup -s backup >> /var/log/incus-copy.log 2>&1 ; restic-incus-backup.sh >> /var/log/restic-incus.log 2>&1
 #
 # Usage: restic-incus-backup.sh [-r <repo>] [-p <project>]
 #          [-x <exclude,list>] [-i <only,these>]
 #
 # All instances by default; opt-out via -x (logged loudly — the list
 # cannot rot silently).
 set -u
 REPO=s3:s3.sbg.io.cloud.ovh.net/restic-incus
 PROJECT=backup
 EXCLUDE_INSTANCES="nextcloud,seafile"   # seek-bound giants: data covered by restic-data
 ONLY_INSTANCES=""
 MNT_ROOT=/run/restic-incus
 LOCKFILE=/run/lock/restic-incus-backup.lock
 ENVFILE=/root/.restic-env
 RESTIC=/usr/local/bin/restic   # cron PATH lacks /usr/local/bin
 MOUNT_TIMEOUT=30
 usage() {
    echo "Usage: $0 [-r <repo>] [-p <project>] [-x <exclude,list>] [-i <only,list>]" >&2
    exit 2
 }
 while getopts r:p:x:i: flag; do
    case "${flag}" in
        r) REPO=${OPTARG};;
        p) PROJECT=${OPTARG};;
        x) EXCLUDE_INSTANCES=${OPTARG};;
        i) ONLY_INSTANCES=${OPTARG};;
        *) usage;;
    esac
 done
 [ -r "$ENVFILE" ] || { echo "env file $ENVFILE not readable" >&2; exit 2; }
 . "$ENVFILE"
 command -v sshfs >/dev/null || { echo "sshfs not installed (needed by incus file mount)" >&2; exit 2; }
 exec 9>"$LOCKFILE"
 if ! flock -n 9; then
    echo "another restic-incus-backup run holds $LOCKFILE, aborting" >&2
    exit 1
 fi
 log() { echo "[$(date '+%F %T')] $*"; }
 cleanup_mount() {  # $1 = mountpoint, $2 = mount pid
    [ -n "${2:-}" ] && kill "$2" 2>/dev/null
    for _ in 1 2 3 4 5; do
        mountpoint -q "$1" || return 0
        fusermount -u "$1" 2>/dev/null || umount "$1" 2>/dev/null
        sleep 1
    done
    mountpoint -q "$1" && { echo "failed to unmount $1" >&2; return 1; }
    return 0
 }
 rc=0
 mkdir -p "$MNT_ROOT"
 for inst in $(incus list --project "$PROJECT" -c n -f csv); do
    if [ -n "$ONLY_INSTANCES" ]; then
        case ",$ONLY_INSTANCES," in *",$inst,"*) ;; *) continue;; esac
    fi
    case ",$EXCLUDE_INSTANCES," in
        *",$inst,"*) log "SKIP $inst (excluded)"; continue;;
    esac
    mnt="$MNT_ROOT/$inst"
    mkdir -p "$mnt"
    mountpoint -q "$mnt" && cleanup_mount "$mnt" ""   # stale from a killed run
    # instance definition, backed up alongside the tree
    incus config show "$inst" --project "$PROJECT" --expanded > "$MNT_ROOT/$inst.yaml" \
        || { echo "config dump of $inst failed" >&2; rc=1; }
    incus file mount "$inst/" "$mnt" --project "$PROJECT" >/dev/null 2>&1 &
    mpid=$!
    mounted=""
    for _ in $(seq "$MOUNT_TIMEOUT"); do
        mountpoint -q "$mnt" && { mounted=1; break; }
        kill -0 "$mpid" 2>/dev/null || break
        sleep 1
    done
    if [ -z "$mounted" ]; then
        echo "mount of $inst failed" >&2; rc=1
        cleanup_mount "$mnt" "$mpid"
        continue
    fi
    log "backup $inst"
    $RESTIC -r "$REPO" backup \
        --pack-size 64 --read-concurrency 8 -o s3.connections=8 \
        --tag "$inst" "$mnt" "$MNT_ROOT/$inst.yaml" \
        || { echo "backup of $inst failed" >&2; rc=1; }
    cleanup_mount "$mnt" "$mpid" || rc=1
 done
 log "forget: keep 14d/8w/6m"
 $RESTIC -r "$REPO" forget --keep-daily 14 --keep-weekly 8 --keep-monthly 6 || rc=1
 log "done (rc=$rc)"
 exit $rc
@@ -1,54 +0,0 @@
 #!/bin/bash
 #
 # Weekly restic maintenance for both repos (Sunday, offset from the
 # nightly backups — prune takes an EXCLUSIVE lock). Per repo:
 #   - unlock: clear locks left by crashed runs
 #   - prune: --max-unused 10% (our dead-data rate is ~0.1–0.4 GiB/day
 #     vs ~1 T repos, so pruning can skip repacking for months) and
 #     --max-repack-size 4G (bounds any single Sunday's rewrite to
 #     ~10-15 min even after a mass deletion; the rest defers)
 #   - check: structure every week, plus a rotating 1/52 data subset —
 #     a full verification of every byte once a year
 #
 # Usage: restic-maintenance.sh [-r <repo>[,<repo>...]]
 set -u
 REPOS="s3:s3.sbg.io.cloud.ovh.net/restic-data,s3:s3.sbg.io.cloud.ovh.net/restic-incus"
 LOCKFILE=/run/lock/restic-maintenance.lock
 ENVFILE=/root/.restic-env
 RESTIC=/usr/local/bin/restic   # cron PATH lacks /usr/local/bin
 while getopts r: flag; do
    case "${flag}" in
        r) REPOS=${OPTARG};;
        *) echo "Usage: $0 [-r <repo>[,<repo>...]]" >&2; exit 2;;
    esac
 done
 [ -r "$ENVFILE" ] || { echo "env file $ENVFILE not readable" >&2; exit 2; }
 . "$ENVFILE"
 exec 9>"$LOCKFILE"
 if ! flock -n 9; then
    echo "another restic-maintenance run holds $LOCKFILE, aborting" >&2
    exit 1
 fi
 log() { echo "[$(date '+%F %T')] $*"; }
 # rotate the read-data subset weekly: full coverage once a year
 WEEK=$(( ($(date +%s) / 604800) % 52 + 1 ))
 rc=0
 for repo in $(printf '%s' "$REPOS" | tr ',' ' '); do
    log "maintenance: $repo"
    $RESTIC -r "$repo" unlock || rc=1
    $RESTIC -r "$repo" prune --max-unused 10% --max-repack-size 4G --pack-size 64 || rc=1
    $RESTIC -r "$repo" check || rc=1
    log "check --read-data-subset=$WEEK/52"
    $RESTIC -r "$repo" check --read-data-subset="$WEEK/52" || rc=1
 done
 log "done (rc=$rc)"
 exit $rc
@@ -1,10 +0,0 @@
 /var/lib/incus/storage-pools/data/containers/bitwarden/rootfs/opt/bitwarden
 /var/lib/incus/storage-pools/data/containers/gateway/rootfs/var/www
 /var/lib/incus/storage-pools/data/containers/git/rootfs/home/git/projects
 /var/lib/incus/storage-pools/data/containers/login/rootfs/opt/authentik
 /var/lib/incus/storage-pools/data/containers/mail/rootfs/var/vmail
 /var/lib/incus/storage-pools/data/containers/mail/rootfs/var/www
 /var/lib/incus/storage-pools/data/containers/outline/rootfs/var/lib/docker/volumes/outline_storage-data/_data
 /var/lib/incus/storage-pools/data/containers/solar/rootfs/var/www/html/solar
 /var/lib/incus/storage-pools/data/containers/nextcloud/rootfs/nextcloud
 /var/lib/incus/storage-pools/data/containers/seafile/rootfs/opt/seafile
@@ -1,101 +0,0 @@
 #!/bin/bash
 #
 # Restore drill for the S3 backup leg — proves the backups are not just
 # "running" but actually restorable. Non-destructive: reads the repo,
 # writes only under a temporary directory, removes it at the end.
 #
 # Two restores, the two data kinds we back up:
 #   1. a filesystem tree   -> restored, then diffed against the live tree
 #   2. a database dump     -> restored, then sanity-checked (and with
 #                             -d, loaded into a throwaway incus container)
 #
 # Usage: restic-restore-test.sh [-r <repo>] [-t <tree-path>] [-b <db-dump>]
 #                               [-w <workdir>] [-k] [-d]
 #   -k  keep the restored files (default: clean up)
 #   -d  also load the dump into a scratch container (slower, strongest)
 #
 # Env from /root/.restic-env.
 set -u
 REPO=s3:s3.sbg.io.cloud.ovh.net/restic-data
 TREE=/var/lib/incus/storage-pools/data/containers/solar/rootfs/var/www/html/solar
 DUMP=/backup/dumps/mariadb/freshrss/freshrss.sql
 WORK=/backup/restore-test-$(date +%Y%m%d-%H%M%S)
 ENVFILE=/root/.restic-env
 RESTIC=/usr/local/bin/restic
 KEEP=0
 DBTEST=0
 while getopts r:t:b:w:kd flag; do
    case "${flag}" in
        r) REPO=${OPTARG};; t) TREE=${OPTARG};; b) DUMP=${OPTARG};;
        w) WORK=${OPTARG};; k) KEEP=1;; d) DBTEST=1;;
        *) echo "Usage: $0 [-r repo] [-t tree] [-b dump] [-w workdir] [-k] [-d]" >&2; exit 2;;
    esac
 done
 [ -r "$ENVFILE" ] || { echo "env file $ENVFILE not readable (run as root)" >&2; exit 2; }
 . "$ENVFILE"
 log() { echo "[$(date '+%F %T')] $*"; }
 rc=0; verdict() { [ "$1" -eq 0 ] && echo "PASS  $2" || { echo "FAIL  $2"; rc=1; }; }
 mkdir -p "$WORK" || exit 2
 log "repo $REPO"
 $RESTIC -r "$REPO" snapshots --latest 1 --compact || { echo "cannot list snapshots" >&2; exit 1; }
 # ---- 1. filesystem tree ----------------------------------------------------
 log "restoring tree: $TREE"
 $RESTIC -r "$REPO" restore latest --target "$WORK/tree" --include "$TREE" >/dev/null 2>&1
 verdict $? "tree restored"
 if [ -d "$WORK/tree$TREE" ]; then
    files=$(find "$WORK/tree$TREE" -type f | wc -l)
    log "restored $files files; diffing against the live tree"
    # differences are expected if the tree changed since the snapshot —
    # report them, do not fail on them
    if diff -qr "$WORK/tree$TREE" "$TREE" > "$WORK/diff.txt" 2>&1; then
        verdict 0 "restored tree is identical to live ($files files)"
    else
        echo "NOTE  $(wc -l < "$WORK/diff.txt") path(s) differ from live (expected if changed since the snapshot):"
        head -5 "$WORK/diff.txt" | sed 's/^/      /'
        verdict 0 "restored tree readable ($files files)"
    fi
 else
    verdict 1 "restored tree missing at $WORK/tree$TREE"
 fi
 # ---- 2. database dump ------------------------------------------------------
 log "restoring dump: $DUMP"
 $RESTIC -r "$REPO" restore latest --target "$WORK/db" --include "$DUMP" >/dev/null 2>&1
 f="$WORK/db$DUMP"
 if [ -s "$f" ]; then
    tables=$(grep -c "^CREATE TABLE" "$f")
    tail -3 "$f" | grep -q "Dump completed"
    complete=$?
    log "dump: $(du -h "$f" | cut -f1), $tables CREATE TABLE, completed-marker=$([ $complete -eq 0 ] && echo yes || echo NO)"
    verdict $(( complete != 0 || tables == 0 ? 1 : 0 )) "dump restored and well-formed"
    if [ "$DBTEST" -eq 1 ]; then
        ct=restore-drill-$$
        log "loading it into a scratch container ($ct) — this takes a few minutes"
        if incus launch images:debian/13 "$ct" >/dev/null 2>&1 &&
           incus exec "$ct" -- sh -c "DEBIAN_FRONTEND=noninteractive apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq install -y mariadb-server" >/dev/null 2>&1; then
            incus file push "$f" "$ct/root/dump.sql" >/dev/null 2>&1
            incus exec "$ct" -- sh -c "mariadb < /root/dump.sql" && loaded=0 || loaded=1
            got=$(incus exec "$ct" -- mariadb -N -B -e \
                 "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema NOT IN ('mysql','information_schema','performance_schema','sys')" 2>/dev/null)
            log "tables in the restored database: ${got:-0} (dump declared $tables)"
            verdict $(( loaded != 0 || ${got:-0} == 0 ? 1 : 0 )) "dump loads into a live MariaDB"
        else
            verdict 1 "could not prepare the scratch container"
        fi
        incus delete -f "$ct" >/dev/null 2>&1
    fi
 else
    verdict 1 "dump restored at $f"
 fi
 # ---- cleanup ---------------------------------------------------------------
 if [ "$KEEP" -eq 1 ]; then log "keeping $WORK"; else rm -rf "$WORK"; log "cleaned up $WORK"; fi
 log "restore drill done (rc=$rc)"
 exit $rc
@@ -20,10 +20,10 @@ TS=$(date '+%s')
 ZFS=$(which zfs)
 CMD="$ZFS snapshot ${DATASET}@snap${TS}"
 echo "[${DATE}] $CMD"
-$CMD 2>&1
+$CMD 2>&1 > /dev/null
 ZFS_PRUNE_SNAPSHOTS=$(which zfs-prune-snapshots)
-CMD="$ZFS_PRUNE_SNAPSHOTS -p snap $EXPIRATION $DATASET"
+CMD="$ZFS_PRUNE_SNAPSHOTS -p 'snap' $EXPIRATION $DATASET"
 echo "[${DATE}] $CMD"
-$CMD 2>&1
+$CMD 2>&1 > /dev/null
@@ -1,86 +0,0 @@
 #!/bin/bash
 #
 # Pool health watchdog. ZED does not cover everything: a pool that gets
 # SUSPENDED after an I/O failure keeps its vdev marked ONLINE, so
 # statechange-notify.sh never fires, and data-class events are silent
 # unless ZED_NOTIFY_DATA is set. Three separate incidents on nuc went
 # unnoticed for days because of that.
 #
 # This checks `zpool status -x` and mails on TRANSITIONS
 # (healthy -> problem, problem -> healthy), plus a low-rate reminder
 # while a problem persists — otherwise a pool stuck unhealthy (e.g.
 # stale `<metadata>` entries in the error log) parks the watchdog in
 # the alarm state, where it can no longer signal anything NEW.
 # Quiet by default, cannot spam. Run every 15 min from cron.
 #
 # Usage: zpool-health.sh [-m <mail-to>] [-s <state-file>] [-i <hours>] [-t]
 #          -i  hours between reminders while unhealthy (default 24, 0=off)
 #          -t  send a test mail and exit (proves the path works)
 set -u
 MAILTO=root
 STATE=/var/lib/zpool-health.state
 REMIND_H=24
 TEST=0
 HOST=$(hostname -s)
 while getopts m:s:i:t flag; do
    case "${flag}" in
        m) MAILTO=${OPTARG};;
        s) STATE=${OPTARG};;
        i) REMIND_H=${OPTARG};;
        t) TEST=1;;
        *) echo "Usage: $0 [-m <mail-to>] [-s <state-file>] [-i <hours>] [-t]" >&2; exit 2;;
    esac
 done
 notify() {  # subject, body
    if command -v mail >/dev/null 2>&1; then
        printf '%s\n' "$2" | mail -s "$1" "$MAILTO"
    else
        logger -t zpool-health "$1"
        printf '%s\n' "$2" | logger -t zpool-health
    fi
 }
 if [ "$TEST" -eq 1 ]; then
    notify "[$HOST] zpool-health test" "$(zpool status -x 2>&1)"
    echo "test notification sent to $MAILTO"; exit 0
 fi
 status=$(zpool status -x 2>&1)
 if [ "$status" = "all pools are healthy" ]; then
    now=ok
 else
    now=problem
 fi
 # state file: "<status> <epoch of last notification>"
 read -r was last < "$STATE" 2>/dev/null || { was=ok; last=0; }
 [ -n "${last:-}" ] || last=0
 nowsec=$(date +%s)
 remind=0
 if [ "$now" = problem ] && [ "$was" = problem ] && [ "$REMIND_H" -gt 0 ]; then
    [ $(( nowsec - last )) -ge $(( REMIND_H * 3600 )) ] && remind=1
 fi
 if [ "$now" = "$was" ] && [ "$remind" -eq 0 ]; then
    printf '%s %s' "$now" "$last" > "$STATE"      # keep the notify time
    [ "$now" = problem ] && exit 1 || exit 0
 fi
 printf '%s %s' "$now" "$nowsec" > "$STATE"
 if [ "$now" = problem ]; then
    subj="[$HOST] ZFS POOL PROBLEM"
    [ "$remind" -eq 1 ] && subj="[$HOST] ZFS pool STILL unhealthy (${REMIND_H}h reminder)"
    notify "$subj" "$(printf '%s\n\n%s\n' "$status" "$(zpool status -v 2>&1)")"
    echo "$(date '+%F %T') problem: $status" >&2
    exit 1
 else
    notify "[$HOST] ZFS pools healthy again" "$(zpool status 2>&1)"
    echo "$(date '+%F %T') recovered"
 fi
 exit 0
		`@@ -1,2 +0,0 @@`
			`/var/lib/incus/storage-pools/data/containers/nextcloud/rootfs/nextcloud/data/appdata_*/preview`
			`/var/lib/incus/storage-pools/data/containers/nextcloud/rootfs/nextcloud/data/appdata_*/dav-photocache`