zpool-health: remind while unhealthy, so a stuck alarm cannot mask new problems

Transition-only notification has a blind spot spotted on nas: while
stale <metadata> entries keep 'zpool status -x' unhealthy, the
watchdog parks in the alarm state and can never signal anything new.
Now it re-notifies every -i hours (default 24) while the problem
persists.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

incus-backup.db

@@ -30,9 +30,5 @@
     "solar": {
         "DB": ["solar"],
         "FS": ["/var/www/html/solar"]
-    },
+    }
-    "spot": {
-        "DB": ["spot"],
-        "FS": ["/var/www"]
-    },
 }

incus-container-upgrade.sh

@@ -3,12 +3,30 @@
 # apt dist-upgrade of all RUNNING containers (containers only — VMs are
 # excluded, they may not run an agent or apt at all). Containers without
 # apt are skipped. Exits non-zero if any upgrade failed.
+#
+# Also refreshes `user.os` / `user.os-checked` on each container, so
+#   incus list -c n,config:user.os,config:user.os-checked
+# always shows the OS the container actually runs. (`image.description`
+# is deliberately left alone: it records what the instance was created
+# from — 2019 images for most of this fleet — which is history worth
+# keeping, not an inventory.)
+#
+# Usage: incus-container-upgrade.sh [-o]     # -o: only refresh user.os,
+#                                            #     skip the upgrades
 
 set -uo pipefail
 
 INCUS=/usr/bin/incus
 LOG=/var/log/incus-container-upgrade.log
 LOCKFILE=/run/lock/incus-container-upgrade.lock
+OSONLY=0
+
+while getopts o flag; do
+    case "${flag}" in
+        o) OSONLY=1;;
+        *) echo "Usage: $0 [-o]" >&2; exit 2;;
+    esac
+done
 
 # refuse to overlap with a previous, still-running invocation
 exec 9> "$LOCKFILE"
@@ -29,12 +47,25 @@ distUpgrade() {
     $INCUS exec "$CT" --env DEBIAN_FRONTEND=noninteractive -- apt-get -qq -y dist-upgrade
 }
 
+# record the OS the container actually runs; failures here never fail
+# the upgrade (it is metadata, not the job)
+recordOS() {
+    local CT=$1 OS
+    OS=$($INCUS exec "$CT" -- sh -c '. /etc/os-release 2>/dev/null && echo "$PRETTY_NAME"' 2>/dev/null)
+    [ -n "$OS" ] || return 0
+    $INCUS config set "$CT" user.os="$OS" user.os-checked="$(date +%F)" 2>/dev/null \
+        || echo "$CT: could not record user.os" >&2
+}
+
 RC=0
 for CT in $($INCUS list -c n -f csv status=RUNNING type=container); do
-    if ! distUpgrade "$CT" 2>&1 | tee -a "$LOG"; then
+    if [ "$OSONLY" -eq 0 ]; then
-        echo "[$(date '+%F %T')] FAILED: $CT" | tee -a "$LOG" >&2
+        if ! distUpgrade "$CT" 2>&1 | tee -a "$LOG"; then
-        RC=1
+            echo "[$(date '+%F %T')] FAILED: $CT" | tee -a "$LOG" >&2
+            RC=1
+        fi
     fi
+    recordOS "$CT"
 done
 
 exit $RC

incus-copy.sh

@@ -64,8 +64,9 @@ INSTANCES=$($INCUS list ${SRC:+"${SRC}:"} -c n -f csv) || {
 RC=0
 for CT in $INSTANCES; do
     echo "[$(date '+%F %T')] copy ${SRC:+$SRC:}$CT -> ${DEST:+$DEST:}$CT${PROJECT:+ (project $PROJECT)}"
-    # --mode only applies to remote transfers
+    # --mode only applies to remote transfers; --quiet suppresses the
-    if $INCUS copy "${SRC:+$SRC:}$CT" "${DEST:+$DEST:}$CT" \
+    # \r-progress meter that turns log files into mangled one-liners
+    if $INCUS copy --quiet "${SRC:+$SRC:}$CT" "${DEST:+$DEST:}$CT" \
         --refresh --refresh-exclude-older \
         ${SRC:+--mode "$MODE"} ${DEST:+--mode "$MODE"} \
         ${POOL:+--storage "$POOL"} \

plakar-backup.sh

@@ -0,0 +1,180 @@
+#!/bin/bash
+#
+# Nightly plakar backup driver. Three phases:
+#   1. dumps — incus's own DBs, plus application-consistent database
+#      dumps taken with `incus exec` (container-local auth: no DB
+#      users, no network exposure). Auto-discovered in every RUNNING
+#      container — no manifest to go stale: native MariaDB
+#      (mariadb-dump, all non-system DBs), native PostgreSQL (pg_dump
+#      per DB + pg_dumpall --globals-only, peer auth as postgres),
+#      and PostgreSQL inside docker containers (image name matching
+#      "postgres", user from $POSTGRES_USER). Dumps are staged plain
+#      (not gzipped, so plakar's chunking dedups near-identical
+#      consecutive dumps) in the dump dir, then backed up as one
+#      snapshot.
+#   2. sources — every fs source listed in the sources file (one
+#      `plakar source add` name per line, #-comments allowed).
+#   3. retention — prune + maintenance on the kloset.
+# The kloset target is whatever `plakar at` accepts — normally the S3
+# store (`-k @s3`, direct-to-S3, no local staging); a path
+# (e.g. -k /backup/plakar-test) only for testing. -S optionally
+# replicates the kloset to a second store afterwards (`plakar sync`);
+# unused in the current single-store setup.
+#
+# Usage: plakar-backup.sh -k <kloset> [-f <sources-file>]
+#          [-r <retention-days>] [-S <sync-store>] [-d <dump-dir>]
+#
+# The kloset passphrase comes from $PLAKAR_PASSPHRASE or, failing
+# that, /root/.plakar-passphrase (mode 600).
+
+set -u
+
+KLOSET=""
+SRC_FILE=/root/scripts/plakar-sources
+# optional DB exclude list, one dump path per line as printed by the
+# "dump" log lines (<ct>/<db> or <ct>/<docker-name>/<db>); every skip
+# is logged loudly so the list can't rot silently like a manifest
+EXCLUDE_FILE=/root/scripts/plakar-db-exclude
+RETENTION=30
+SYNC_STORE=""
+DUMP_DIR=/backup/plakar-dumps
+LOCKFILE=/run/lock/plakar-backup.lock
+PASSFILE=/root/.plakar-passphrase
+
+usage() {
+    echo "Usage: $0 -k <kloset> [-f <sources-file>] [-r <retention-days>] [-S <sync-store>] [-d <dump-dir>]" >&2
+    exit 2
+}
+
+while getopts k:f:r:S:d: flag; do
+    case "${flag}" in
+        k) KLOSET=${OPTARG};;
+        f) SRC_FILE=${OPTARG};;
+        r) RETENTION=${OPTARG};;
+        S) SYNC_STORE=${OPTARG};;
+        d) DUMP_DIR=${OPTARG};;
+        *) usage;;
+    esac
+done
+
+[ -n "$KLOSET" ] || usage
+[ -r "$SRC_FILE" ] || { echo "sources file $SRC_FILE not readable" >&2; exit 2; }
+
+exec 9>"$LOCKFILE"
+if ! flock -n 9; then
+    echo "another plakar-backup run holds $LOCKFILE, aborting" >&2
+    exit 1
+fi
+
+log() { echo "[$(date '+%F %T')] $*"; }
+
+excluded() {
+    [ -r "$EXCLUDE_FILE" ] && grep -qx "$1" "$EXCLUDE_FILE" \
+        && log "SKIP $1 (listed in $EXCLUDE_FILE)"
+}
+
+if [ -z "${PLAKAR_PASSPHRASE:-}" ]; then
+    [ -r "$PASSFILE" ] || { echo "no PLAKAR_PASSPHRASE in env and $PASSFILE not readable" >&2; exit 1; }
+    PLAKAR_PASSPHRASE=$(cat "$PASSFILE")
+fi
+export PLAKAR_PASSPHRASE
+
+rc=0
+
+# incus's own state (instance configs, profiles, devices) — needed to
+# rebuild instances, not covered by any file/DB source
+mkdir -p "$DUMP_DIR/incus"
+incus admin sql global .dump > "$DUMP_DIR/incus/incus-global-db.sql" || rc=1
+incus admin sql local .dump > "$DUMP_DIR/incus/incus-local-db.sql" || rc=1
+
+# Database dumps, auto-discovered per running container: native
+# MariaDB, native PostgreSQL, and PostgreSQL inside docker containers
+# (matched on the docker image name containing "postgres").
+PG_LIST="SELECT datname FROM pg_database WHERE NOT datistemplate AND datname <> 'postgres'"
+for ct in $(incus list status=running -c n -f csv); do
+    # native MariaDB/MySQL (unix-socket root auth) — binaries may carry
+    # either naming (e.g. mail: hand-installed mariadb-dump + mysql client)
+    mdump=$(incus exec "$ct" -- sh -c 'command -v mariadb-dump || command -v mysqldump' 2>/dev/null)
+    mclient=$(incus exec "$ct" -- sh -c 'command -v mariadb || command -v mysql' 2>/dev/null)
+    if [ -n "$mdump" ] && [ -n "$mclient" ]; then
+        dbs=$(incus exec "$ct" -- "$mclient" -N -B -e 'SHOW DATABASES') \
+            || { echo "listing mariadb databases on $ct failed" >&2; rc=1; dbs=""; }
+        for db in $(printf '%s\n' "$dbs" \
+                    | grep -Ev '^(information_schema|performance_schema|mysql|sys)$'); do
+            excluded "$ct/$db" && continue
+            log "dump $ct/$db (mariadb)"
+            mkdir -p "$DUMP_DIR/mariadb/$ct"
+            incus exec "$ct" -- "$mdump" --single-transaction --events --routines --triggers \
+                --databases "$db" > "$DUMP_DIR/mariadb/$ct/$db.sql" \
+                || { echo "dump $ct/$db failed" >&2; rc=1; }
+        done
+        # users + grants (the mariadb equivalent of pg_dumpall
+        # --globals-only): replayable SHOW GRANTS statements, portable
+        # across versions unlike a raw mysql-schema dump
+        log "dump $ct/grants (mariadb)"
+        incus exec "$ct" -- sh -c "$mclient -NBe \"SELECT CONCAT('SHOW GRANTS FOR ', QUOTE(user), '@', QUOTE(host), ';') FROM mysql.user\" | $mclient -NB | sed 's/\$/;/'" \
+            > "$DUMP_DIR/mariadb/$ct/grants.sql" \
+            || { echo "grants dump on $ct failed" >&2; rc=1; }
+    elif [ -n "$mdump$mclient" ]; then
+        echo "$ct has only one of dump/client mariadb binaries, skipping" >&2; rc=1
+    fi
+
+    # native PostgreSQL (peer auth as the postgres user)
+    if incus exec "$ct" -- sh -c 'command -v pg_dump' >/dev/null 2>&1; then
+        mkdir -p "$DUMP_DIR/postgres/$ct"
+        incus exec "$ct" -- su -s /bin/sh postgres -c "pg_dumpall --globals-only" \
+            > "$DUMP_DIR/postgres/$ct/globals.sql" || rc=1
+        dbs=$(incus exec "$ct" -- su -s /bin/sh postgres -c "psql -AtX -c \"$PG_LIST\"") \
+            || { echo "listing postgres databases on $ct failed" >&2; rc=1; dbs=""; }
+        for db in $dbs; do
+            excluded "$ct/$db" && continue
+            log "dump $ct/$db (postgres)"
+            incus exec "$ct" -- su -s /bin/sh postgres -c "pg_dump --clean --if-exists $db" \
+                > "$DUMP_DIR/postgres/$ct/$db.sql" \
+                || { echo "dump $ct/$db failed" >&2; rc=1; }
+        done
+    fi
+
+    # PostgreSQL inside docker (e.g. outline, login)
+    incus exec "$ct" -- sh -c 'command -v docker' >/dev/null 2>&1 || continue
+    for dc in $(incus exec "$ct" -- docker ps --format '{{.Names}} {{.Image}}' 2>/dev/null \
+                | awk 'tolower($2) ~ /postgres/ {print $1}'); do
+        pguser=$(incus exec "$ct" -- docker exec "$dc" sh -c 'echo "${POSTGRES_USER:-postgres}"') \
+            || { echo "reading POSTGRES_USER on $ct/$dc failed" >&2; rc=1; continue; }
+        mkdir -p "$DUMP_DIR/postgres/$ct/$dc"
+        incus exec "$ct" -- docker exec "$dc" pg_dumpall -U "$pguser" --globals-only \
+            > "$DUMP_DIR/postgres/$ct/$dc/globals.sql" || rc=1
+        dbs=$(incus exec "$ct" -- docker exec "$dc" psql -U "$pguser" -AtX -c "$PG_LIST") \
+            || { echo "listing postgres databases on $ct/$dc failed" >&2; rc=1; dbs=""; }
+        for db in $dbs; do
+            excluded "$ct/$dc/$db" && continue
+            log "dump $ct/$dc/$db (postgres)"
+            incus exec "$ct" -- docker exec "$dc" pg_dump -U "$pguser" --clean --if-exists "$db" \
+                > "$DUMP_DIR/postgres/$ct/$dc/$db.sql" \
+                || { echo "dump $ct/$dc/$db failed" >&2; rc=1; }
+        done
+    done
+done
+
+log "backup dumps ($DUMP_DIR)"
+plakar -quiet at "$KLOSET" backup -tag dumps "$DUMP_DIR" \
+    || { echo "backup of $DUMP_DIR failed" >&2; rc=1; }
+
+for src in $(grep -Ev '^[[:space:]]*(#|$)' "$SRC_FILE"); do
+    log "backup @$src"
+    plakar -quiet at "$KLOSET" backup -tag "$src" "@$src" \
+        || { echo "backup @$src failed" >&2; rc=1; }
+done
+
+log "prune: keep the last $RETENTION days of snapshots"
+plakar at "$KLOSET" prune -days "$RETENTION" -apply || rc=1
+plakar at "$KLOSET" maintenance || rc=1
+
+if [ -n "$SYNC_STORE" ]; then
+    log "sync to @$SYNC_STORE"
+    plakar at "$KLOSET" sync to "@$SYNC_STORE" \
+        || { echo "sync to @$SYNC_STORE failed" >&2; rc=1; }
+fi
+
+log "done (rc=$rc)"
+exit $rc

plakar-incus-backup.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+#
+# Nightly plakar backup of the incus instances (leg 4): backs up the
+# configured incus source (the plakar incus integration walking the
+# quiesced replicas of the `backup` project) into the target kloset,
+# then applies retention. Runs CHAINED after incus-copy.sh in the same
+# cron entry — the snapshot is only as fresh as the last completed
+# local replica refresh:
+#
+#   0 1 * * * incus-copy.sh -p backup -s backup >> /var/log/incus-copy.log 2>&1 ; plakar-incus-backup.sh >> /var/log/plakar-incus.log 2>&1
+#
+# Usage: plakar-incus-backup.sh [-k <kloset>] [-s <source>] [-r <retention-days>]
+#
+# The kloset passphrase comes from $PLAKAR_PASSPHRASE or, failing
+# that, /root/.plakar-passphrase (mode 600).
+
+set -u
+
+KLOSET=@s3incus
+SOURCE=ks4-incus
+RETENTION=30
+LOCKFILE=/run/lock/plakar-incus-backup.lock
+PASSFILE=/root/.plakar-passphrase
+
+usage() {
+    echo "Usage: $0 [-k <kloset>] [-s <source>] [-r <retention-days>]" >&2
+    exit 2
+}
+
+while getopts k:s:r: flag; do
+    case "${flag}" in
+        k) KLOSET=${OPTARG};;
+        s) SOURCE=${OPTARG};;
+        r) RETENTION=${OPTARG};;
+        *) usage;;
+    esac
+done
+
+exec 9>"$LOCKFILE"
+if ! flock -n 9; then
+    echo "another plakar-incus-backup run holds $LOCKFILE, aborting" >&2
+    exit 1
+fi
+
+log() { echo "[$(date '+%F %T')] $*"; }
+
+if [ -z "${PLAKAR_PASSPHRASE:-}" ]; then
+    [ -r "$PASSFILE" ] || { echo "no PLAKAR_PASSPHRASE in env and $PASSFILE not readable" >&2; exit 1; }
+    PLAKAR_PASSPHRASE=$(cat "$PASSFILE")
+fi
+export PLAKAR_PASSPHRASE
+
+rc=0
+
+# The plakar pkg backend has been observed to silently expel an
+# installed plugin (its reload path deletes the ptar when the cache
+# re-extraction fails — seen 2026-08-24). Self-heal from the kept
+# ptar instead of failing the nightly run with
+# "unsupported importer protocol".
+if ! plakar pkg list 2>/dev/null | grep -q "^incus@"; then
+    PTAR=$(ls -t /root/incus-plugin/incus_v*.ptar 2>/dev/null | head -1)
+    echo "incus plugin missing — reinstalling ${PTAR:-<no ptar found>}" >&2
+    [ -n "$PTAR" ] && plakar pkg add "$PTAR" \
+        || { echo "incus plugin reinstall failed" >&2; exit 1; }
+fi
+
+log "backup @$SOURCE -> $KLOSET"
+plakar -quiet at "$KLOSET" backup -tag "$SOURCE" "@$SOURCE" \
+    || { echo "backup @$SOURCE failed" >&2; rc=1; }
+
+log "prune: keep the last $RETENTION days of snapshots"
+plakar at "$KLOSET" prune -days "$RETENTION" -apply || rc=1
+plakar at "$KLOSET" maintenance || rc=1
+
+log "done (rc=$rc)"
+exit $rc

restic-backup.sh

@@ -0,0 +1,170 @@
+#!/bin/bash
+#
+# Nightly restic data backup (repo: restic-data). Three phases:
+#   1. dumps — incus's own DBs, plus application-consistent database
+#      dumps taken with `incus exec` (container-local auth: no DB
+#      users, no network exposure). Auto-discovered in every RUNNING
+#      container — no manifest to go stale: native MariaDB/MySQL
+#      (either binary naming), native PostgreSQL (pg_dump per DB +
+#      pg_dumpall --globals-only), and PostgreSQL inside docker
+#      containers (image name matching "postgres"). Dumps staged
+#      plain (not gzipped — CDC dedup needs uncompressed input).
+#   2. one `restic backup` invocation: the dump dir + every path in
+#      the paths file (one index load, one snapshot per night);
+#      exclude patterns applied globally.
+#   3. retention — `restic forget --group-by host` (seed-era
+#      snapshots have different path sets and must age in one group).
+#      Prune/check live in restic-maintenance.sh (weekly).
+#
+# Usage: restic-backup.sh [-r <repo>] [-f <paths-file>] [-d <dump-dir>]
+#          [-s <stage>]   # stage: dumps|backup|all (default all);
+#                         # -s backup skips re-dumping (seeding aid)
+#
+# Env from /root/.restic-env (AWS creds, RESTIC_PASSWORD_FILE,
+# RESTIC_CACHE_DIR).
+
+set -u
+
+REPO=s3:s3.sbg.io.cloud.ovh.net/restic-data
+PATHS_FILE=/root/scripts/restic-paths
+EXCLUDE_FILE=/root/scripts/restic-exclude
+DB_EXCLUDE_FILE=/root/scripts/db-exclude
+# legacy name from the plakar era — keep working until the file is renamed
+[ -r "$DB_EXCLUDE_FILE" ] || [ ! -r /root/scripts/plakar-db-exclude ] || {
+    DB_EXCLUDE_FILE=/root/scripts/plakar-db-exclude
+    echo "note: using legacy $DB_EXCLUDE_FILE — rename it to /root/scripts/db-exclude" >&2
+}
+DUMP_DIR=/backup/dumps
+LOCKFILE=/run/lock/restic-backup.lock
+ENVFILE=/root/.restic-env
+RESTIC=/usr/local/bin/restic   # cron PATH lacks /usr/local/bin
+STAGE=all
+
+usage() {
+    echo "Usage: $0 [-r <repo>] [-f <paths-file>] [-d <dump-dir>] [-s dumps|backup|all]" >&2
+    exit 2
+}
+
+while getopts r:f:d:s: flag; do
+    case "${flag}" in
+        r) REPO=${OPTARG};;
+        f) PATHS_FILE=${OPTARG};;
+        d) DUMP_DIR=${OPTARG};;
+        s) STAGE=${OPTARG};;
+        *) usage;;
+    esac
+done
+
+[ -r "$PATHS_FILE" ] || { echo "paths file $PATHS_FILE not readable" >&2; exit 2; }
+[ -r "$ENVFILE" ] || { echo "env file $ENVFILE not readable" >&2; exit 2; }
+. "$ENVFILE"
+
+exec 9>"$LOCKFILE"
+if ! flock -n 9; then
+    echo "another restic-backup run holds $LOCKFILE, aborting" >&2
+    exit 1
+fi
+
+log() { echo "[$(date '+%F %T')] $*"; }
+
+excluded() {
+    [ -r "$DB_EXCLUDE_FILE" ] && grep -qx "$1" "$DB_EXCLUDE_FILE" \
+        && log "SKIP $1 (listed in $DB_EXCLUDE_FILE)"
+}
+
+rc=0
+
+if [ "$STAGE" = all ] || [ "$STAGE" = dumps ]; then
+
+# incus's own state (instance configs, profiles, devices)
+mkdir -p "$DUMP_DIR/incus"
+incus admin sql global .dump > "$DUMP_DIR/incus/incus-global-db.sql" || rc=1
+incus admin sql local .dump > "$DUMP_DIR/incus/incus-local-db.sql" || rc=1
+
+# Database dumps, auto-discovered per running container
+PG_LIST="SELECT datname FROM pg_database WHERE NOT datistemplate AND datname <> 'postgres'"
+for ct in $(incus list status=running -c n -f csv); do
+    # native MariaDB/MySQL (unix-socket root auth), either binary naming
+    mdump=$(incus exec "$ct" -- sh -c 'command -v mariadb-dump || command -v mysqldump' 2>/dev/null)
+    mclient=$(incus exec "$ct" -- sh -c 'command -v mariadb || command -v mysql' 2>/dev/null)
+    if [ -n "$mdump" ] && [ -n "$mclient" ]; then
+        dbs=$(incus exec "$ct" -- "$mclient" -N -B -e 'SHOW DATABASES') \
+            || { echo "listing mariadb databases on $ct failed" >&2; rc=1; dbs=""; }
+        for db in $(printf '%s\n' "$dbs" \
+                    | grep -Ev '^(information_schema|performance_schema|mysql|sys)$'); do
+            excluded "$ct/$db" && continue
+            log "dump $ct/$db (mariadb)"
+            mkdir -p "$DUMP_DIR/mariadb/$ct"
+            incus exec "$ct" -- "$mdump" --single-transaction --events --routines --triggers \
+                --databases "$db" > "$DUMP_DIR/mariadb/$ct/$db.sql" \
+                || { echo "dump $ct/$db failed" >&2; rc=1; }
+        done
+        # users + grants: replayable SHOW GRANTS statements
+        log "dump $ct/grants (mariadb)"
+        incus exec "$ct" -- sh -c "$mclient -NBe \"SELECT CONCAT('SHOW GRANTS FOR ', QUOTE(user), '@', QUOTE(host), ';') FROM mysql.user\" | $mclient -NB | sed 's/\$/;/'" \
+            > "$DUMP_DIR/mariadb/$ct/grants.sql" \
+            || { echo "grants dump on $ct failed" >&2; rc=1; }
+    elif [ -n "$mdump$mclient" ]; then
+        echo "$ct has only one of dump/client mariadb binaries, skipping" >&2; rc=1
+    fi
+
+    # native PostgreSQL (peer auth as the postgres user)
+    if incus exec "$ct" -- sh -c 'command -v pg_dump' >/dev/null 2>&1; then
+        mkdir -p "$DUMP_DIR/postgres/$ct"
+        incus exec "$ct" -- su -s /bin/sh postgres -c "pg_dumpall --globals-only" \
+            > "$DUMP_DIR/postgres/$ct/globals.sql" || rc=1
+        dbs=$(incus exec "$ct" -- su -s /bin/sh postgres -c "psql -AtX -c \"$PG_LIST\"") \
+            || { echo "listing postgres databases on $ct failed" >&2; rc=1; dbs=""; }
+        for db in $dbs; do
+            excluded "$ct/$db" && continue
+            log "dump $ct/$db (postgres)"
+            incus exec "$ct" -- su -s /bin/sh postgres -c "pg_dump --clean --if-exists $db" \
+                > "$DUMP_DIR/postgres/$ct/$db.sql" \
+                || { echo "dump $ct/$db failed" >&2; rc=1; }
+        done
+    fi
+
+    # PostgreSQL inside docker (e.g. outline, login)
+    incus exec "$ct" -- sh -c 'command -v docker' >/dev/null 2>&1 || continue
+    for dc in $(incus exec "$ct" -- docker ps --format '{{.Names}} {{.Image}}' 2>/dev/null \
+                | awk 'tolower($2) ~ /postgres/ {print $1}'); do
+        pguser=$(incus exec "$ct" -- docker exec "$dc" sh -c 'echo "${POSTGRES_USER:-postgres}"') \
+            || { echo "reading POSTGRES_USER on $ct/$dc failed" >&2; rc=1; continue; }
+        mkdir -p "$DUMP_DIR/postgres/$ct/$dc"
+        incus exec "$ct" -- docker exec "$dc" pg_dumpall -U "$pguser" --globals-only \
+            > "$DUMP_DIR/postgres/$ct/$dc/globals.sql" || rc=1
+        dbs=$(incus exec "$ct" -- docker exec "$dc" psql -U "$pguser" -AtX -c "$PG_LIST") \
+            || { echo "listing postgres databases on $ct/$dc failed" >&2; rc=1; dbs=""; }
+        for db in $dbs; do
+            excluded "$ct/$dc/$db" && continue
+            log "dump $ct/$dc/$db (postgres)"
+            incus exec "$ct" -- docker exec "$dc" pg_dump -U "$pguser" --clean --if-exists "$db" \
+                > "$DUMP_DIR/postgres/$ct/$dc/$db.sql" \
+                || { echo "dump $ct/$dc/$db failed" >&2; rc=1; }
+        done
+    done
+done
+
+fi  # stage dumps
+
+if [ "$STAGE" = all ] || [ "$STAGE" = backup ]; then
+
+# clear locks left by a killed run (safe: only stale ones are removed);
+# otherwise a single interrupted backup blocks forget/prune for ever
+$RESTIC -r "$REPO" unlock >/dev/null 2>&1 || true
+
+log "restic backup -> $REPO"
+$RESTIC -r "$REPO" backup \
+    --pack-size 64 --read-concurrency 8 -o s3.connections=8 \
+    --exclude-file "$EXCLUDE_FILE" \
+    --files-from-verbatim "$PATHS_FILE" "$DUMP_DIR" \
+    || { echo "restic backup failed" >&2; rc=1; }
+
+log "forget: keep 14d/8w/6m"
+$RESTIC -r "$REPO" forget --group-by host \
+    --keep-daily 14 --keep-weekly 8 --keep-monthly 6 || rc=1
+
+fi  # stage backup
+
+log "done (rc=$rc)"
+exit $rc

restic-exclude

@@ -0,0 +1,2 @@
+/var/lib/incus/storage-pools/data/containers/nextcloud/rootfs/nextcloud/data/appdata_*/preview
+/var/lib/incus/storage-pools/data/containers/nextcloud/rootfs/nextcloud/data/appdata_*/dav-photocache

restic-incus-backup.sh

@@ -0,0 +1,121 @@
+#!/bin/bash
+#
+# Nightly restic backup of the incus instances (repo: restic-incus).
+# For every instance of the `backup` project (quiesced replicas —
+# stopped, refreshed by the 01:00 incus-copy), mount its filesystem
+# with `incus file mount` (FUSE over the per-instance sftp API, works
+# on stopped containers, needs sshfs) and back it up per-file,
+# together with its expanded config.
+#
+# ⚠️ Each instance gets its OWN mountpoint (/run/restic-incus/<name>):
+# restic selects a snapshot's parent by host+path, so a shared
+# mountpoint would parent every snapshot on the previous *other*
+# instance and force nightly full re-reads.
+#
+# Runs CHAINED after incus-copy.sh in the same cron entry — the
+# snapshot is only as fresh as the last completed replica refresh:
+#   0 1 * * * incus-copy.sh -p backup -s backup >> /var/log/incus-copy.log 2>&1 ; restic-incus-backup.sh >> /var/log/restic-incus.log 2>&1
+#
+# Usage: restic-incus-backup.sh [-r <repo>] [-p <project>]
+#          [-x <exclude,list>] [-i <only,these>]
+#
+# All instances by default; opt-out via -x (logged loudly — the list
+# cannot rot silently).
+
+set -u
+
+REPO=s3:s3.sbg.io.cloud.ovh.net/restic-incus
+PROJECT=backup
+EXCLUDE_INSTANCES="nextcloud,seafile"   # seek-bound giants: data covered by restic-data
+ONLY_INSTANCES=""
+MNT_ROOT=/run/restic-incus
+LOCKFILE=/run/lock/restic-incus-backup.lock
+ENVFILE=/root/.restic-env
+RESTIC=/usr/local/bin/restic   # cron PATH lacks /usr/local/bin
+MOUNT_TIMEOUT=30
+
+usage() {
+    echo "Usage: $0 [-r <repo>] [-p <project>] [-x <exclude,list>] [-i <only,list>]" >&2
+    exit 2
+}
+
+while getopts r:p:x:i: flag; do
+    case "${flag}" in
+        r) REPO=${OPTARG};;
+        p) PROJECT=${OPTARG};;
+        x) EXCLUDE_INSTANCES=${OPTARG};;
+        i) ONLY_INSTANCES=${OPTARG};;
+        *) usage;;
+    esac
+done
+
+[ -r "$ENVFILE" ] || { echo "env file $ENVFILE not readable" >&2; exit 2; }
+. "$ENVFILE"
+command -v sshfs >/dev/null || { echo "sshfs not installed (needed by incus file mount)" >&2; exit 2; }
+
+exec 9>"$LOCKFILE"
+if ! flock -n 9; then
+    echo "another restic-incus-backup run holds $LOCKFILE, aborting" >&2
+    exit 1
+fi
+
+log() { echo "[$(date '+%F %T')] $*"; }
+
+cleanup_mount() {  # $1 = mountpoint, $2 = mount pid
+    [ -n "${2:-}" ] && kill "$2" 2>/dev/null
+    for _ in 1 2 3 4 5; do
+        mountpoint -q "$1" || return 0
+        fusermount -u "$1" 2>/dev/null || umount "$1" 2>/dev/null
+        sleep 1
+    done
+    mountpoint -q "$1" && { echo "failed to unmount $1" >&2; return 1; }
+    return 0
+}
+
+rc=0
+mkdir -p "$MNT_ROOT"
+
+for inst in $(incus list --project "$PROJECT" -c n -f csv); do
+    if [ -n "$ONLY_INSTANCES" ]; then
+        case ",$ONLY_INSTANCES," in *",$inst,"*) ;; *) continue;; esac
+    fi
+    case ",$EXCLUDE_INSTANCES," in
+        *",$inst,"*) log "SKIP $inst (excluded)"; continue;;
+    esac
+
+    mnt="$MNT_ROOT/$inst"
+    mkdir -p "$mnt"
+    mountpoint -q "$mnt" && cleanup_mount "$mnt" ""   # stale from a killed run
+
+    # instance definition, backed up alongside the tree
+    incus config show "$inst" --project "$PROJECT" --expanded > "$MNT_ROOT/$inst.yaml" \
+        || { echo "config dump of $inst failed" >&2; rc=1; }
+
+    incus file mount "$inst/" "$mnt" --project "$PROJECT" >/dev/null 2>&1 &
+    mpid=$!
+    mounted=""
+    for _ in $(seq "$MOUNT_TIMEOUT"); do
+        mountpoint -q "$mnt" && { mounted=1; break; }
+        kill -0 "$mpid" 2>/dev/null || break
+        sleep 1
+    done
+    if [ -z "$mounted" ]; then
+        echo "mount of $inst failed" >&2; rc=1
+        cleanup_mount "$mnt" "$mpid"
+        continue
+    fi
+
+    log "backup $inst"
+    $RESTIC -r "$REPO" backup \
+        --pack-size 64 --read-concurrency 8 -o s3.connections=8 \
+        --tag "$inst" "$mnt" "$MNT_ROOT/$inst.yaml" \
+        || { echo "backup of $inst failed" >&2; rc=1; }
+
+    cleanup_mount "$mnt" "$mpid" || rc=1
+done
+
+log "forget: keep 14d/8w/6m"
+$RESTIC -r "$REPO" forget --keep-daily 14 --keep-weekly 8 --keep-monthly 6 || rc=1
+
+log "done (rc=$rc)"
+exit $rc

restic-maintenance.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+#
+# Weekly restic maintenance for both repos (Sunday, offset from the
+# nightly backups — prune takes an EXCLUSIVE lock). Per repo:
+#   - unlock: clear locks left by crashed runs
+#   - prune: --max-unused 10% (our dead-data rate is ~0.1–0.4 GiB/day
+#     vs ~1 T repos, so pruning can skip repacking for months) and
+#     --max-repack-size 4G (bounds any single Sunday's rewrite to
+#     ~10-15 min even after a mass deletion; the rest defers)
+#   - check: structure every week, plus a rotating 1/52 data subset —
+#     a full verification of every byte once a year
+#
+# Usage: restic-maintenance.sh [-r <repo>[,<repo>...]]
+
+set -u
+
+REPOS="s3:s3.sbg.io.cloud.ovh.net/restic-data,s3:s3.sbg.io.cloud.ovh.net/restic-incus"
+LOCKFILE=/run/lock/restic-maintenance.lock
+ENVFILE=/root/.restic-env
+RESTIC=/usr/local/bin/restic   # cron PATH lacks /usr/local/bin
+
+while getopts r: flag; do
+    case "${flag}" in
+        r) REPOS=${OPTARG};;
+        *) echo "Usage: $0 [-r <repo>[,<repo>...]]" >&2; exit 2;;
+    esac
+done
+
+[ -r "$ENVFILE" ] || { echo "env file $ENVFILE not readable" >&2; exit 2; }
+. "$ENVFILE"
+
+exec 9>"$LOCKFILE"
+if ! flock -n 9; then
+    echo "another restic-maintenance run holds $LOCKFILE, aborting" >&2
+    exit 1
+fi
+
+log() { echo "[$(date '+%F %T')] $*"; }
+
+# rotate the read-data subset weekly: full coverage once a year
+WEEK=$(( ($(date +%s) / 604800) % 52 + 1 ))
+
+rc=0
+for repo in $(printf '%s' "$REPOS" | tr ',' ' '); do
+    log "maintenance: $repo"
+    $RESTIC -r "$repo" unlock || rc=1
+    $RESTIC -r "$repo" prune --max-unused 10% --max-repack-size 4G --pack-size 64 || rc=1
+    $RESTIC -r "$repo" check || rc=1
+    log "check --read-data-subset=$WEEK/52"
+    $RESTIC -r "$repo" check --read-data-subset="$WEEK/52" || rc=1
+done
+
+log "done (rc=$rc)"
+exit $rc

restic-paths

@@ -0,0 +1,10 @@
+/var/lib/incus/storage-pools/data/containers/bitwarden/rootfs/opt/bitwarden
+/var/lib/incus/storage-pools/data/containers/gateway/rootfs/var/www
+/var/lib/incus/storage-pools/data/containers/git/rootfs/home/git/projects
+/var/lib/incus/storage-pools/data/containers/login/rootfs/opt/authentik
+/var/lib/incus/storage-pools/data/containers/mail/rootfs/var/vmail
+/var/lib/incus/storage-pools/data/containers/mail/rootfs/var/www
+/var/lib/incus/storage-pools/data/containers/outline/rootfs/var/lib/docker/volumes/outline_storage-data/_data
+/var/lib/incus/storage-pools/data/containers/solar/rootfs/var/www/html/solar
+/var/lib/incus/storage-pools/data/containers/nextcloud/rootfs/nextcloud
+/var/lib/incus/storage-pools/data/containers/seafile/rootfs/opt/seafile

restic-restore-test.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+#
+# Restore drill for the S3 backup leg — proves the backups are not just
+# "running" but actually restorable. Non-destructive: reads the repo,
+# writes only under a temporary directory, removes it at the end.
+#
+# Two restores, the two data kinds we back up:
+#   1. a filesystem tree   -> restored, then diffed against the live tree
+#   2. a database dump     -> restored, then sanity-checked (and with
+#                             -d, loaded into a throwaway incus container)
+#
+# Usage: restic-restore-test.sh [-r <repo>] [-t <tree-path>] [-b <db-dump>]
+#                               [-w <workdir>] [-k] [-d]
+#   -k  keep the restored files (default: clean up)
+#   -d  also load the dump into a scratch container (slower, strongest)
+#
+# Env from /root/.restic-env.
+
+set -u
+
+REPO=s3:s3.sbg.io.cloud.ovh.net/restic-data
+TREE=/var/lib/incus/storage-pools/data/containers/solar/rootfs/var/www/html/solar
+DUMP=/backup/dumps/mariadb/freshrss/freshrss.sql
+WORK=/backup/restore-test-$(date +%Y%m%d-%H%M%S)
+ENVFILE=/root/.restic-env
+RESTIC=/usr/local/bin/restic
+KEEP=0
+DBTEST=0
+
+while getopts r:t:b:w:kd flag; do
+    case "${flag}" in
+        r) REPO=${OPTARG};; t) TREE=${OPTARG};; b) DUMP=${OPTARG};;
+        w) WORK=${OPTARG};; k) KEEP=1;; d) DBTEST=1;;
+        *) echo "Usage: $0 [-r repo] [-t tree] [-b dump] [-w workdir] [-k] [-d]" >&2; exit 2;;
+    esac
+done
+
+[ -r "$ENVFILE" ] || { echo "env file $ENVFILE not readable (run as root)" >&2; exit 2; }
+. "$ENVFILE"
+log() { echo "[$(date '+%F %T')] $*"; }
+rc=0; verdict() { [ "$1" -eq 0 ] && echo "PASS  $2" || { echo "FAIL  $2"; rc=1; }; }
+
+mkdir -p "$WORK" || exit 2
+log "repo $REPO"
+$RESTIC -r "$REPO" snapshots --latest 1 --compact || { echo "cannot list snapshots" >&2; exit 1; }
+
+# ---- 1. filesystem tree ----------------------------------------------------
+log "restoring tree: $TREE"
+$RESTIC -r "$REPO" restore latest --target "$WORK/tree" --include "$TREE" >/dev/null 2>&1
+verdict $? "tree restored"
+if [ -d "$WORK/tree$TREE" ]; then
+    files=$(find "$WORK/tree$TREE" -type f | wc -l)
+    log "restored $files files; diffing against the live tree"
+    # differences are expected if the tree changed since the snapshot —
+    # report them, do not fail on them
+    if diff -qr "$WORK/tree$TREE" "$TREE" > "$WORK/diff.txt" 2>&1; then
+        verdict 0 "restored tree is identical to live ($files files)"
+    else
+        echo "NOTE  $(wc -l < "$WORK/diff.txt") path(s) differ from live (expected if changed since the snapshot):"
+        head -5 "$WORK/diff.txt" | sed 's/^/      /'
+        verdict 0 "restored tree readable ($files files)"
+    fi
+else
+    verdict 1 "restored tree missing at $WORK/tree$TREE"
+fi
+
+# ---- 2. database dump ------------------------------------------------------
+log "restoring dump: $DUMP"
+$RESTIC -r "$REPO" restore latest --target "$WORK/db" --include "$DUMP" >/dev/null 2>&1
+f="$WORK/db$DUMP"
+if [ -s "$f" ]; then
+    tables=$(grep -c "^CREATE TABLE" "$f")
+    tail -3 "$f" | grep -q "Dump completed"
+    complete=$?
+    log "dump: $(du -h "$f" | cut -f1), $tables CREATE TABLE, completed-marker=$([ $complete -eq 0 ] && echo yes || echo NO)"
+    verdict $(( complete != 0 || tables == 0 ? 1 : 0 )) "dump restored and well-formed"
+
+    if [ "$DBTEST" -eq 1 ]; then
+        ct=restore-drill-$$
+        log "loading it into a scratch container ($ct) — this takes a few minutes"
+        if incus launch images:debian/13 "$ct" >/dev/null 2>&1 &&
+           incus exec "$ct" -- sh -c "DEBIAN_FRONTEND=noninteractive apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq install -y mariadb-server" >/dev/null 2>&1; then
+            incus file push "$f" "$ct/root/dump.sql" >/dev/null 2>&1
+            incus exec "$ct" -- sh -c "mariadb < /root/dump.sql" && loaded=0 || loaded=1
+            got=$(incus exec "$ct" -- mariadb -N -B -e \
+                 "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema NOT IN ('mysql','information_schema','performance_schema','sys')" 2>/dev/null)
+            log "tables in the restored database: ${got:-0} (dump declared $tables)"
+            verdict $(( loaded != 0 || ${got:-0} == 0 ? 1 : 0 )) "dump loads into a live MariaDB"
+        else
+            verdict 1 "could not prepare the scratch container"
+        fi
+        incus delete -f "$ct" >/dev/null 2>&1
+    fi
+else
+    verdict 1 "dump restored at $f"
+fi
+
+# ---- cleanup ---------------------------------------------------------------
+if [ "$KEEP" -eq 1 ]; then log "keeping $WORK"; else rm -rf "$WORK"; log "cleaned up $WORK"; fi
+log "restore drill done (rc=$rc)"
+exit $rc

zpool-health.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+#
+# Pool health watchdog. ZED does not cover everything: a pool that gets
+# SUSPENDED after an I/O failure keeps its vdev marked ONLINE, so
+# statechange-notify.sh never fires, and data-class events are silent
+# unless ZED_NOTIFY_DATA is set. Three separate incidents on nuc went
+# unnoticed for days because of that.
+#
+# This checks `zpool status -x` and mails on TRANSITIONS
+# (healthy -> problem, problem -> healthy), plus a low-rate reminder
+# while a problem persists — otherwise a pool stuck unhealthy (e.g.
+# stale `<metadata>` entries in the error log) parks the watchdog in
+# the alarm state, where it can no longer signal anything NEW.
+# Quiet by default, cannot spam. Run every 15 min from cron.
+#
+# Usage: zpool-health.sh [-m <mail-to>] [-s <state-file>] [-i <hours>] [-t]
+#          -i  hours between reminders while unhealthy (default 24, 0=off)
+#          -t  send a test mail and exit (proves the path works)
+
+set -u
+
+MAILTO=root
+STATE=/var/lib/zpool-health.state
+REMIND_H=24
+TEST=0
+HOST=$(hostname -s)
+
+while getopts m:s:i:t flag; do
+    case "${flag}" in
+        m) MAILTO=${OPTARG};;
+        s) STATE=${OPTARG};;
+        i) REMIND_H=${OPTARG};;
+        t) TEST=1;;
+        *) echo "Usage: $0 [-m <mail-to>] [-s <state-file>] [-i <hours>] [-t]" >&2; exit 2;;
+    esac
+done
+
+notify() {  # subject, body
+    if command -v mail >/dev/null 2>&1; then
+        printf '%s\n' "$2" | mail -s "$1" "$MAILTO"
+    else
+        logger -t zpool-health "$1"
+        printf '%s\n' "$2" | logger -t zpool-health
+    fi
+}
+
+if [ "$TEST" -eq 1 ]; then
+    notify "[$HOST] zpool-health test" "$(zpool status -x 2>&1)"
+    echo "test notification sent to $MAILTO"; exit 0
+fi
+
+status=$(zpool status -x 2>&1)
+if [ "$status" = "all pools are healthy" ]; then
+    now=ok
+else
+    now=problem
+fi
+
+# state file: "<status> <epoch of last notification>"
+read -r was last < "$STATE" 2>/dev/null || { was=ok; last=0; }
+[ -n "${last:-}" ] || last=0
+nowsec=$(date +%s)
+
+remind=0
+if [ "$now" = problem ] && [ "$was" = problem ] && [ "$REMIND_H" -gt 0 ]; then
+    [ $(( nowsec - last )) -ge $(( REMIND_H * 3600 )) ] && remind=1
+fi
+
+if [ "$now" = "$was" ] && [ "$remind" -eq 0 ]; then
+    printf '%s %s' "$now" "$last" > "$STATE"      # keep the notify time
+    [ "$now" = problem ] && exit 1 || exit 0
+fi
+
+printf '%s %s' "$now" "$nowsec" > "$STATE"
+
+if [ "$now" = problem ]; then
+    subj="[$HOST] ZFS POOL PROBLEM"
+    [ "$remind" -eq 1 ] && subj="[$HOST] ZFS pool STILL unhealthy (${REMIND_H}h reminder)"
+    notify "$subj" "$(printf '%s\n\n%s\n' "$status" "$(zpool status -v 2>&1)")"
+    echo "$(date '+%F %T') problem: $status" >&2
+    exit 1
+else
+    notify "[$HOST] ZFS pools healthy again" "$(zpool status 2>&1)"
+    echo "$(date '+%F %T') recovered"
+fi
+exit 0