Add zfs-auto-snapshot.sh script

README.md

@@ -3,7 +3,6 @@
 Some helpful bash scripts for [incus](https://linuxcontainers.org/incus)
 
 - incus-container-upgrade: Perform `apt dist-upgrade` command on running instances.
-- incus-backup: Backup instances FS and DBs to a local directory (`-d local`) or a remote location using rsync over ssh.
-- incus-copy: Run a differential copy of all instances to a remote incus server (push or pull) or to a local project/pool.
+- incus-backup: Backup instances FS and DBs to a remote location using rsync over ssh.
+- incus-copy: Run a differential copy of all running instances to a remote incus server.
 - incus-snapshot: Take a snapshot on all running instances.
-- zfs-auto-snapshot: Create and cleanup expired zfs snapshots.

incus-backup.db

@@ -30,5 +30,12 @@
     "solar": {
         "DB": ["solar"],
         "FS": ["/var/www/html/solar"]
+    },
+    "spot": {
+        "DB": ["spot"],
+        "FS": ["/var/www"]
+    },
+    "wedding": {
+        "FS": ["/var/www/mariage"]
     }
 }

incus-backup.sh

@@ -1,106 +1,55 @@
 #!/bin/bash
 #
-# Backup instances FS and DBs, driven by a JSON manifest (see
-# incus-backup.db: {"<instance>": {"DB": [...], "FS": [...]}, ...}).
-# Destination is either a remote (rsync over ssh) or, with -d local,
-# a local directory (e.g. a dedicated backup zpool mounted on /backup).
+# Backup instances FS and DBs to a remote location using rsync over ssh.
 #
-# Usage: incus-backup.sh -f <manifest> -s <storage-pool>
-#          [-d <dest-host|local>] [-u <ssh-user>] [-i <ssh-key>]
-#          [-p <ssh-port>] [-b <backup-root>]
-#
-# Layout under <backup-root> (default /backup): <hostname>/<instance>/
-#   plus <hostname>/incus-{local,global}-db.sql
 
-set -u
-
-DEST=local
-SSH_USER=""
-SSH_KEY=""
-SSH_PORT=22
-DB_FILE=""
-STORAGE_POOL=""
-BKP_ROOT=/backup
-LOCKFILE=/run/lock/incus-backup.lock
-
-usage() {
-    echo "Usage: $0 -f <manifest.db> -s <storage-pool> [-d <dest-host|local>] [-u <ssh-user>] [-i <ssh-key>] [-p <ssh-port>] [-b <backup-root>]" >&2
-    exit 2
-}
-
-while getopts d:u:i:p:f:s:b: flag; do
+while getopts d:u:i:p:f:s: flag
+do
     case "${flag}" in
         d) DEST=${OPTARG};;
-        u) SSH_USER=${OPTARG};;
+        u) USER=${OPTARG};;
         i) SSH_KEY=${OPTARG};;
         p) SSH_PORT=${OPTARG};;
         f) DB_FILE=${OPTARG};;
         s) STORAGE_POOL=${OPTARG};;
-        b) BKP_ROOT=${OPTARG};;
-        *) usage;;
     esac
 done
-[ -n "$DB_FILE" ] && [ -n "$STORAGE_POOL" ] || usage
-if [ "$DEST" != "local" ]; then
-    [ -n "$SSH_USER" ] && [ -n "$SSH_KEY" ] || usage
-fi
 
 HOST=$(hostname -s)
-BKP_DIR="${BKP_ROOT}/${HOST}"
+BKP_DIR="/backup/${HOST}"
+SSH_CMD="ssh -i ${SSH_KEY} -p ${SSH_PORT}"
 CT_PREFIX="/var/lib/incus/storage-pools/${STORAGE_POOL}/containers"
 
-# save <dst-file>          — write stdin to a file at the destination
-# transfer <opts> <src> <dst-dir>  — rsync a path to the destination
-if [ "$DEST" = "local" ]; then
-    save()     { mkdir -p "$(dirname "$1")" && cat - > "$1"; }
-    transfer() { mkdir -p "$3" && /usr/bin/rsync $1 "$2" "$3"; }
-else
-    SSH_CMD="ssh -i ${SSH_KEY} -p ${SSH_PORT}"
-    save()     { ${SSH_CMD} "${SSH_USER}@${DEST}" "mkdir -p $(dirname "$1") && cat - > $1"; }
-    transfer() { /usr/bin/rsync $1 -e "${SSH_CMD}" "$2" "${SSH_USER}@${DEST}:$3"; }
-fi
-
-# refuse to overlap with a previous, still-running invocation
-exec 9> "$LOCKFILE"
-if ! flock -n 9; then
-    echo "another incus-backup run holds $LOCKFILE, aborting" >&2
-    exit 1
-fi
-
-RC=0
 
 # Backup incus DB
-/usr/bin/incus admin sql local .dump  | save "${BKP_DIR}/incus-local-db.sql"  || RC=1
-/usr/bin/incus admin sql global .dump | save "${BKP_DIR}/incus-global-db.sql" || RC=1
+/usr/bin/incus admin sql local .dump | ${SSH_CMD} ${USER}@${DEST} "cat - > ${BKP_DIR}/incus-local-db.sql"
+/usr/bin/incus admin sql global .dump | ${SSH_CMD} ${USER}@${DEST} "cat - > ${BKP_DIR}/incus-global-db.sql"
 
-for CT in $(jq -r 'keys[]' "${DB_FILE}") ; do
+for CT in $(cat ${DB_FILE} | jq -r 'keys[]') ; do
     SRC_DIR="${CT_PREFIX}/${CT}"
     DST_DIR="${BKP_DIR}/${CT}"
 
     # Backup container info
     if [ -f "${SRC_DIR}/backup.yaml" ] ; then
-        echo "[$(date '+%F %T')] Starting $CT backup.yaml"
-        transfer "-a --del" "${SRC_DIR}/backup.yaml" "${DST_DIR}/" || RC=1
+        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting $CT backup.yaml"
+        /usr/bin/rsync -a --del -e "${SSH_CMD}" ${SRC_DIR}/backup.yaml ${USER}@${DEST}:${DST_DIR}/
     fi
 
     # Backup Mysql dumps
-    for DB in $(jq -r ".${CT} | select(.DB != null) | .DB[]" "${DB_FILE}") ; do
-        echo "[$(date '+%F %T')] Starting $CT $DB database backup"
-        /usr/bin/incus exec "$CT" -- mariadb-dump --single-transaction --databases "${DB}" \
-            | save "${DST_DIR}/mysql-${DB}.sql" || RC=1
+    for DB in $(cat $DB_FILE | jq -r ".${CT} | select(.DB != null) | .DB[]") ; do
+        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting $CT $DB database backup"
+        /usr/bin/incus exec -n $CT -- mariadb-dump --single-transaction --databases ${DB} | ${SSH_CMD} ${USER}@${DEST} "mkdir -p ${DST_DIR} ; cat - > ${DST_DIR}/mysql-${DB}.sql"
     done
 
-    # Backup container rootfs paths
-    for FS in $(jq -r ".${CT} | select(.FS != null) | .FS[]" "${DB_FILE}") ; do
+    # Backup container rootfs paths 
+    for FS in $(cat $DB_FILE | jq -r ".${CT} | select(.FS != null) | .FS[]") ; do
         # Skip missing rootfs dir
-        if [ ! -d "${SRC_DIR}/rootfs" ] ; then
-            echo "[$(date '+%F %T')] WARNING : Missing rootfs for container ${CT}, skipping FS ${FS} ..."
+        if [ ! -d ${SRC_DIR}/rootfs ] ; then
+            echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARNING : Missing rootfs for container ${CT}, skipping FS ${FS} ..."
             continue
         fi
-        # "/." used by rsync to limit the amount of path information that is sent as implied directories
-        echo "[$(date '+%F %T')] Starting $CT $FS filesystem backup"
-        transfer "-aR --del" "${SRC_DIR}/rootfs/.${FS}" "${DST_DIR}/" || RC=1
+	    # "/." used by rsync to limit the amount of path information that is sent as implied directories
+        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting $CT $FS filesystem backup"
+	    /usr/bin/rsync -aR --del -e "${SSH_CMD}" ${SRC_DIR}/rootfs/.${FS} ${USER}@${DEST}:${DST_DIR}/
     done
 done
-
-exit $RC

incus-container-upgrade.sh

@@ -1,71 +1,15 @@
 #!/bin/bash
 #
-# apt dist-upgrade of all RUNNING containers (containers only — VMs are
-# excluded, they may not run an agent or apt at all). Containers without
-# apt are skipped. Exits non-zero if any upgrade failed.
-#
-# Also refreshes `user.os` / `user.os-checked` on each container, so
-#   incus list -c n,config:user.os,config:user.os-checked
-# always shows the OS the container actually runs. (`image.description`
-# is deliberately left alone: it records what the instance was created
-# from — 2019 images for most of this fleet — which is history worth
-# keeping, not an inventory.)
-#
-# Usage: incus-container-upgrade.sh [-o]     # -o: only refresh user.os,
-#                                            #     skip the upgrades
 
-set -uo pipefail
-
-INCUS=/usr/bin/incus
-LOG=/var/log/incus-container-upgrade.log
-LOCKFILE=/run/lock/incus-container-upgrade.lock
-OSONLY=0
-
-while getopts o flag; do
-    case "${flag}" in
-        o) OSONLY=1;;
-        *) echo "Usage: $0 [-o]" >&2; exit 2;;
-    esac
-done
-
-# refuse to overlap with a previous, still-running invocation
-exec 9> "$LOCKFILE"
-if ! flock -n 9; then
-    echo "another incus-container-upgrade run holds $LOCKFILE, aborting" >&2
-    exit 1
-fi
 
 distUpgrade() {
-    local CT=$1
-    echo -e "\n*** [$(date '+%F %T')] - Dist-upgrading $CT container ***\n"
-    if ! $INCUS exec "$CT" -- sh -c 'command -v apt-get >/dev/null'; then
-        echo "$CT: no apt-get in container, skipping"
-        return 0
-    fi
-    $INCUS exec "$CT" --env DEBIAN_FRONTEND=noninteractive -- apt-get -qq update &&
-    $INCUS exec "$CT" -- sh -c 'apt list --upgradable 2>/dev/null' &&
-    $INCUS exec "$CT" --env DEBIAN_FRONTEND=noninteractive -- apt-get -qq -y dist-upgrade
+    CT=$1
+    DATE=$(date '+%Y-%m-%d %H:%M:%S')
+    echo -e "\n*** [$DATE] - Dist-upgrading $CT container ***\n"
+    incus exec $CT --env DEBIAN_FRONTEND=noninteractive -- apt -qq update
+    incus exec $CT --env DEBIAN_FRONTEND=noninteractive -- apt -qq -y dist-upgrade
 }
 
-# record the OS the container actually runs; failures here never fail
-# the upgrade (it is metadata, not the job)
-recordOS() {
-    local CT=$1 OS
-    OS=$($INCUS exec "$CT" -- sh -c '. /etc/os-release 2>/dev/null && echo "$PRETTY_NAME"' 2>/dev/null)
-    [ -n "$OS" ] || return 0
-    $INCUS config set "$CT" user.os="$OS" user.os-checked="$(date +%F)" 2>/dev/null \
-        || echo "$CT: could not record user.os" >&2
-}
-
-RC=0
-for CT in $($INCUS list -c n -f csv status=RUNNING type=container); do
-    if [ "$OSONLY" -eq 0 ]; then
-        if ! distUpgrade "$CT" 2>&1 | tee -a "$LOG"; then
-            echo "[$(date '+%F %T')] FAILED: $CT" | tee -a "$LOG" >&2
-            RC=1
-        fi
-    fi
-    recordOS "$CT"
+for CT in $(incus ls -c n -f compact,noheader status=RUNNING) ; do
+    distUpgrade "$CT" | tee -a /var/log/incus-container-upgrade.log
 done
-
-exit $RC

incus-copy.sh

@@ -1,86 +1,23 @@
 #!/bin/bash
 #
-# Incremental replication of ALL Incus instances (incus copy --refresh;
-# with ZFS on both ends only deltas travel, so stopped instances cost
-# nothing after their first copy).
 # https://linuxcontainers.org/incus/docs/main/howto/move_instances/
 #
-# Three shapes, per where it runs and where the replicas land:
-#   incus-copy.sh -d <dest-remote> [-m pull|push|relay] [-s <dest-pool>]
-#       push all local instances to a remote (runs on the source host)
-#   incus-copy.sh -r <src-remote> [-m pull|push|relay] [-s <dest-pool>]
-#       pull all of a remote's instances to the local host (runs on the
-#       backup host; the source only needs its API reachable)
-#   incus-copy.sh -p <target-project> -s <dest-pool>
-#       copy all local instances into another local project + pool
-#       (on-host backup onto a different disk)
-#
-# Snapshot history/retention is handled on the source instances
-# (snapshots.schedule / snapshots.expiry); --refresh-exclude-older keeps
-# refreshes from dragging expired snapshots around.
 
-set -u
+#set -x
 
-INCUS=/usr/bin/incus
-LOCKFILE=/run/lock/incus-copy.lock
-SRC=""
-DEST=""
-MODE=pull
-POOL=""
-PROJECT=""
-
-usage() {
-    echo "Usage: $0 [-d <dest-remote> | -r <src-remote>] [-m pull|push|relay] [-s <dest-storage-pool>] [-p <dest-project>]" >&2
-    exit 2
-}
-
-while getopts d:r:m:s:p: flag; do
+while getopts d:m: flag
+do
     case "${flag}" in
         d) DEST=${OPTARG};;
-        r) SRC=${OPTARG};;
         m) MODE=${OPTARG};;
-        s) POOL=${OPTARG};;
-        p) PROJECT=${OPTARG};;
-        *) usage;;
     esac
 done
-# need somewhere for replicas to go, and one side must stay local
-[ -n "$DEST" ] || [ -n "$SRC" ] || [ -n "$PROJECT" ] || usage
-[ -n "$DEST" ] && [ -n "$SRC" ] && usage
-case "$MODE" in pull|push|relay) ;; *) usage;; esac
 
-# refuse to overlap with a previous, still-running invocation
-exec 9> "$LOCKFILE"
-if ! flock -n 9; then
-    echo "another incus-copy run holds $LOCKFILE, aborting" >&2
-    exit 1
-fi
 
-INSTANCES=$($INCUS list ${SRC:+"${SRC}:"} -c n -f csv) || {
-    echo "cannot list instances${SRC:+ on ${SRC}:}, aborting" >&2
-    exit 1
-}
-
-RC=0
-for CT in $INSTANCES; do
-    echo "[$(date '+%F %T')] copy ${SRC:+$SRC:}$CT -> ${DEST:+$DEST:}$CT${PROJECT:+ (project $PROJECT)}"
-    # --mode only applies to remote transfers; --quiet suppresses the
-    # \r-progress meter that turns log files into mangled one-liners
-    if $INCUS copy --quiet "${SRC:+$SRC:}$CT" "${DEST:+$DEST:}$CT" \
-        --refresh --refresh-exclude-older \
-        ${SRC:+--mode "$MODE"} ${DEST:+--mode "$MODE"} \
-        ${POOL:+--storage "$POOL"} \
-        ${PROJECT:+--target-project "$PROJECT"}; then
-        # the replica inherits boot.autostart from the source and would
-        # start itself on the backup host after a reboot (or, for the
-        # local-project shape, fight the live instance for its static
-        # IP on the same bridge) — neutralize it
-        $INCUS config set ${PROJECT:+--project "$PROJECT"} \
-            "${DEST:+$DEST:}$CT" boot.autostart=false || RC=1
-    else
-        echo "[$(date '+%F %T')] FAILED: $CT" >&2
-        RC=1
-    fi
+for CT in $(/usr/bin/incus list -c n -f compact,noheader status=RUNNING) ; do
+    DATE=$(date '+%Y-%m-%d %H:%M:%S')
+    CMD="/usr/bin/incus copy $CT ${DEST}:$CT --refresh --refresh-exclude-older --mode ${MODE}"
+    echo "[${DATE}] $CMD"
+    $CMD 2>&1 > /dev/null
 done
 
-exit $RC

plakar-backup.sh

@@ -1,180 +0,0 @@
-#!/bin/bash
-#
-# Nightly plakar backup driver. Three phases:
-#   1. dumps — incus's own DBs, plus application-consistent database
-#      dumps taken with `incus exec` (container-local auth: no DB
-#      users, no network exposure). Auto-discovered in every RUNNING
-#      container — no manifest to go stale: native MariaDB
-#      (mariadb-dump, all non-system DBs), native PostgreSQL (pg_dump
-#      per DB + pg_dumpall --globals-only, peer auth as postgres),
-#      and PostgreSQL inside docker containers (image name matching
-#      "postgres", user from $POSTGRES_USER). Dumps are staged plain
-#      (not gzipped, so plakar's chunking dedups near-identical
-#      consecutive dumps) in the dump dir, then backed up as one
-#      snapshot.
-#   2. sources — every fs source listed in the sources file (one
-#      `plakar source add` name per line, #-comments allowed).
-#   3. retention — prune + maintenance on the kloset.
-# The kloset target is whatever `plakar at` accepts — normally the S3
-# store (`-k @s3`, direct-to-S3, no local staging); a path
-# (e.g. -k /backup/plakar-test) only for testing. -S optionally
-# replicates the kloset to a second store afterwards (`plakar sync`);
-# unused in the current single-store setup.
-#
-# Usage: plakar-backup.sh -k <kloset> [-f <sources-file>]
-#          [-r <retention-days>] [-S <sync-store>] [-d <dump-dir>]
-#
-# The kloset passphrase comes from $PLAKAR_PASSPHRASE or, failing
-# that, /root/.plakar-passphrase (mode 600).
-
-set -u
-
-KLOSET=""
-SRC_FILE=/root/scripts/plakar-sources
-# optional DB exclude list, one dump path per line as printed by the
-# "dump" log lines (<ct>/<db> or <ct>/<docker-name>/<db>); every skip
-# is logged loudly so the list can't rot silently like a manifest
-EXCLUDE_FILE=/root/scripts/plakar-db-exclude
-RETENTION=30
-SYNC_STORE=""
-DUMP_DIR=/backup/plakar-dumps
-LOCKFILE=/run/lock/plakar-backup.lock
-PASSFILE=/root/.plakar-passphrase
-
-usage() {
-    echo "Usage: $0 -k <kloset> [-f <sources-file>] [-r <retention-days>] [-S <sync-store>] [-d <dump-dir>]" >&2
-    exit 2
-}
-
-while getopts k:f:r:S:d: flag; do
-    case "${flag}" in
-        k) KLOSET=${OPTARG};;
-        f) SRC_FILE=${OPTARG};;
-        r) RETENTION=${OPTARG};;
-        S) SYNC_STORE=${OPTARG};;
-        d) DUMP_DIR=${OPTARG};;
-        *) usage;;
-    esac
-done
-
-[ -n "$KLOSET" ] || usage
-[ -r "$SRC_FILE" ] || { echo "sources file $SRC_FILE not readable" >&2; exit 2; }
-
-exec 9>"$LOCKFILE"
-if ! flock -n 9; then
-    echo "another plakar-backup run holds $LOCKFILE, aborting" >&2
-    exit 1
-fi
-
-log() { echo "[$(date '+%F %T')] $*"; }
-
-excluded() {
-    [ -r "$EXCLUDE_FILE" ] && grep -qx "$1" "$EXCLUDE_FILE" \
-        && log "SKIP $1 (listed in $EXCLUDE_FILE)"
-}
-
-if [ -z "${PLAKAR_PASSPHRASE:-}" ]; then
-    [ -r "$PASSFILE" ] || { echo "no PLAKAR_PASSPHRASE in env and $PASSFILE not readable" >&2; exit 1; }
-    PLAKAR_PASSPHRASE=$(cat "$PASSFILE")
-fi
-export PLAKAR_PASSPHRASE
-
-rc=0
-
-# incus's own state (instance configs, profiles, devices) — needed to
-# rebuild instances, not covered by any file/DB source
-mkdir -p "$DUMP_DIR/incus"
-incus admin sql global .dump > "$DUMP_DIR/incus/incus-global-db.sql" || rc=1
-incus admin sql local .dump > "$DUMP_DIR/incus/incus-local-db.sql" || rc=1
-
-# Database dumps, auto-discovered per running container: native
-# MariaDB, native PostgreSQL, and PostgreSQL inside docker containers
-# (matched on the docker image name containing "postgres").
-PG_LIST="SELECT datname FROM pg_database WHERE NOT datistemplate AND datname <> 'postgres'"
-for ct in $(incus list status=running -c n -f csv); do
-    # native MariaDB/MySQL (unix-socket root auth) — binaries may carry
-    # either naming (e.g. mail: hand-installed mariadb-dump + mysql client)
-    mdump=$(incus exec "$ct" -- sh -c 'command -v mariadb-dump || command -v mysqldump' 2>/dev/null)
-    mclient=$(incus exec "$ct" -- sh -c 'command -v mariadb || command -v mysql' 2>/dev/null)
-    if [ -n "$mdump" ] && [ -n "$mclient" ]; then
-        dbs=$(incus exec "$ct" -- "$mclient" -N -B -e 'SHOW DATABASES') \
-            || { echo "listing mariadb databases on $ct failed" >&2; rc=1; dbs=""; }
-        for db in $(printf '%s\n' "$dbs" \
-                    | grep -Ev '^(information_schema|performance_schema|mysql|sys)$'); do
-            excluded "$ct/$db" && continue
-            log "dump $ct/$db (mariadb)"
-            mkdir -p "$DUMP_DIR/mariadb/$ct"
-            incus exec "$ct" -- "$mdump" --single-transaction --events --routines --triggers \
-                --databases "$db" > "$DUMP_DIR/mariadb/$ct/$db.sql" \
-                || { echo "dump $ct/$db failed" >&2; rc=1; }
-        done
-        # users + grants (the mariadb equivalent of pg_dumpall
-        # --globals-only): replayable SHOW GRANTS statements, portable
-        # across versions unlike a raw mysql-schema dump
-        log "dump $ct/grants (mariadb)"
-        incus exec "$ct" -- sh -c "$mclient -NBe \"SELECT CONCAT('SHOW GRANTS FOR ', QUOTE(user), '@', QUOTE(host), ';') FROM mysql.user\" | $mclient -NB | sed 's/\$/;/'" \
-            > "$DUMP_DIR/mariadb/$ct/grants.sql" \
-            || { echo "grants dump on $ct failed" >&2; rc=1; }
-    elif [ -n "$mdump$mclient" ]; then
-        echo "$ct has only one of dump/client mariadb binaries, skipping" >&2; rc=1
-    fi
-
-    # native PostgreSQL (peer auth as the postgres user)
-    if incus exec "$ct" -- sh -c 'command -v pg_dump' >/dev/null 2>&1; then
-        mkdir -p "$DUMP_DIR/postgres/$ct"
-        incus exec "$ct" -- su -s /bin/sh postgres -c "pg_dumpall --globals-only" \
-            > "$DUMP_DIR/postgres/$ct/globals.sql" || rc=1
-        dbs=$(incus exec "$ct" -- su -s /bin/sh postgres -c "psql -AtX -c \"$PG_LIST\"") \
-            || { echo "listing postgres databases on $ct failed" >&2; rc=1; dbs=""; }
-        for db in $dbs; do
-            excluded "$ct/$db" && continue
-            log "dump $ct/$db (postgres)"
-            incus exec "$ct" -- su -s /bin/sh postgres -c "pg_dump --clean --if-exists $db" \
-                > "$DUMP_DIR/postgres/$ct/$db.sql" \
-                || { echo "dump $ct/$db failed" >&2; rc=1; }
-        done
-    fi
-
-    # PostgreSQL inside docker (e.g. outline, login)
-    incus exec "$ct" -- sh -c 'command -v docker' >/dev/null 2>&1 || continue
-    for dc in $(incus exec "$ct" -- docker ps --format '{{.Names}} {{.Image}}' 2>/dev/null \
-                | awk 'tolower($2) ~ /postgres/ {print $1}'); do
-        pguser=$(incus exec "$ct" -- docker exec "$dc" sh -c 'echo "${POSTGRES_USER:-postgres}"') \
-            || { echo "reading POSTGRES_USER on $ct/$dc failed" >&2; rc=1; continue; }
-        mkdir -p "$DUMP_DIR/postgres/$ct/$dc"
-        incus exec "$ct" -- docker exec "$dc" pg_dumpall -U "$pguser" --globals-only \
-            > "$DUMP_DIR/postgres/$ct/$dc/globals.sql" || rc=1
-        dbs=$(incus exec "$ct" -- docker exec "$dc" psql -U "$pguser" -AtX -c "$PG_LIST") \
-            || { echo "listing postgres databases on $ct/$dc failed" >&2; rc=1; dbs=""; }
-        for db in $dbs; do
-            excluded "$ct/$dc/$db" && continue
-            log "dump $ct/$dc/$db (postgres)"
-            incus exec "$ct" -- docker exec "$dc" pg_dump -U "$pguser" --clean --if-exists "$db" \
-                > "$DUMP_DIR/postgres/$ct/$dc/$db.sql" \
-                || { echo "dump $ct/$dc/$db failed" >&2; rc=1; }
-        done
-    done
-done
-
-log "backup dumps ($DUMP_DIR)"
-plakar -quiet at "$KLOSET" backup -tag dumps "$DUMP_DIR" \
-    || { echo "backup of $DUMP_DIR failed" >&2; rc=1; }
-
-for src in $(grep -Ev '^[[:space:]]*(#|$)' "$SRC_FILE"); do
-    log "backup @$src"
-    plakar -quiet at "$KLOSET" backup -tag "$src" "@$src" \
-        || { echo "backup @$src failed" >&2; rc=1; }
-done
-
-log "prune: keep the last $RETENTION days of snapshots"
-plakar at "$KLOSET" prune -days "$RETENTION" -apply || rc=1
-plakar at "$KLOSET" maintenance || rc=1
-
-if [ -n "$SYNC_STORE" ]; then
-    log "sync to @$SYNC_STORE"
-    plakar at "$KLOSET" sync to "@$SYNC_STORE" \
-        || { echo "sync to @$SYNC_STORE failed" >&2; rc=1; }
-fi
-
-log "done (rc=$rc)"
-exit $rc

plakar-incus-backup.sh

@@ -1,76 +0,0 @@
-#!/bin/bash
-#
-# Nightly plakar backup of the incus instances (leg 4): backs up the
-# configured incus source (the plakar incus integration walking the
-# quiesced replicas of the `backup` project) into the target kloset,
-# then applies retention. Runs CHAINED after incus-copy.sh in the same
-# cron entry — the snapshot is only as fresh as the last completed
-# local replica refresh:
-#
-#   0 1 * * * incus-copy.sh -p backup -s backup >> /var/log/incus-copy.log 2>&1 ; plakar-incus-backup.sh >> /var/log/plakar-incus.log 2>&1
-#
-# Usage: plakar-incus-backup.sh [-k <kloset>] [-s <source>] [-r <retention-days>]
-#
-# The kloset passphrase comes from $PLAKAR_PASSPHRASE or, failing
-# that, /root/.plakar-passphrase (mode 600).
-
-set -u
-
-KLOSET=@s3incus
-SOURCE=ks4-incus
-RETENTION=30
-LOCKFILE=/run/lock/plakar-incus-backup.lock
-PASSFILE=/root/.plakar-passphrase
-
-usage() {
-    echo "Usage: $0 [-k <kloset>] [-s <source>] [-r <retention-days>]" >&2
-    exit 2
-}
-
-while getopts k:s:r: flag; do
-    case "${flag}" in
-        k) KLOSET=${OPTARG};;
-        s) SOURCE=${OPTARG};;
-        r) RETENTION=${OPTARG};;
-        *) usage;;
-    esac
-done
-
-exec 9>"$LOCKFILE"
-if ! flock -n 9; then
-    echo "another plakar-incus-backup run holds $LOCKFILE, aborting" >&2
-    exit 1
-fi
-
-log() { echo "[$(date '+%F %T')] $*"; }
-
-if [ -z "${PLAKAR_PASSPHRASE:-}" ]; then
-    [ -r "$PASSFILE" ] || { echo "no PLAKAR_PASSPHRASE in env and $PASSFILE not readable" >&2; exit 1; }
-    PLAKAR_PASSPHRASE=$(cat "$PASSFILE")
-fi
-export PLAKAR_PASSPHRASE
-
-rc=0
-
-# The plakar pkg backend has been observed to silently expel an
-# installed plugin (its reload path deletes the ptar when the cache
-# re-extraction fails — seen 2026-08-24). Self-heal from the kept
-# ptar instead of failing the nightly run with
-# "unsupported importer protocol".
-if ! plakar pkg list 2>/dev/null | grep -q "^incus@"; then
-    PTAR=$(ls -t /root/incus-plugin/incus_v*.ptar 2>/dev/null | head -1)
-    echo "incus plugin missing — reinstalling ${PTAR:-<no ptar found>}" >&2
-    [ -n "$PTAR" ] && plakar pkg add "$PTAR" \
-        || { echo "incus plugin reinstall failed" >&2; exit 1; }
-fi
-
-log "backup @$SOURCE -> $KLOSET"
-plakar -quiet at "$KLOSET" backup -tag "$SOURCE" "@$SOURCE" \
-    || { echo "backup @$SOURCE failed" >&2; rc=1; }
-
-log "prune: keep the last $RETENTION days of snapshots"
-plakar at "$KLOSET" prune -days "$RETENTION" -apply || rc=1
-plakar at "$KLOSET" maintenance || rc=1
-
-log "done (rc=$rc)"
-exit $rc

restic-backup.sh

@@ -1,170 +0,0 @@
-#!/bin/bash
-#
-# Nightly restic data backup (repo: restic-data). Three phases:
-#   1. dumps — incus's own DBs, plus application-consistent database
-#      dumps taken with `incus exec` (container-local auth: no DB
-#      users, no network exposure). Auto-discovered in every RUNNING
-#      container — no manifest to go stale: native MariaDB/MySQL
-#      (either binary naming), native PostgreSQL (pg_dump per DB +
-#      pg_dumpall --globals-only), and PostgreSQL inside docker
-#      containers (image name matching "postgres"). Dumps staged
-#      plain (not gzipped — CDC dedup needs uncompressed input).
-#   2. one `restic backup` invocation: the dump dir + every path in
-#      the paths file (one index load, one snapshot per night);
-#      exclude patterns applied globally.
-#   3. retention — `restic forget --group-by host` (seed-era
-#      snapshots have different path sets and must age in one group).
-#      Prune/check live in restic-maintenance.sh (weekly).
-#
-# Usage: restic-backup.sh [-r <repo>] [-f <paths-file>] [-d <dump-dir>]
-#          [-s <stage>]   # stage: dumps|backup|all (default all);
-#                         # -s backup skips re-dumping (seeding aid)
-#
-# Env from /root/.restic-env (AWS creds, RESTIC_PASSWORD_FILE,
-# RESTIC_CACHE_DIR).
-
-set -u
-
-REPO=s3:s3.sbg.io.cloud.ovh.net/restic-data
-PATHS_FILE=/root/scripts/restic-paths
-EXCLUDE_FILE=/root/scripts/restic-exclude
-DB_EXCLUDE_FILE=/root/scripts/db-exclude
-# legacy name from the plakar era — keep working until the file is renamed
-[ -r "$DB_EXCLUDE_FILE" ] || [ ! -r /root/scripts/plakar-db-exclude ] || {
-    DB_EXCLUDE_FILE=/root/scripts/plakar-db-exclude
-    echo "note: using legacy $DB_EXCLUDE_FILE — rename it to /root/scripts/db-exclude" >&2
-}
-DUMP_DIR=/backup/dumps
-LOCKFILE=/run/lock/restic-backup.lock
-ENVFILE=/root/.restic-env
-RESTIC=/usr/local/bin/restic   # cron PATH lacks /usr/local/bin
-STAGE=all
-
-usage() {
-    echo "Usage: $0 [-r <repo>] [-f <paths-file>] [-d <dump-dir>] [-s dumps|backup|all]" >&2
-    exit 2
-}
-
-while getopts r:f:d:s: flag; do
-    case "${flag}" in
-        r) REPO=${OPTARG};;
-        f) PATHS_FILE=${OPTARG};;
-        d) DUMP_DIR=${OPTARG};;
-        s) STAGE=${OPTARG};;
-        *) usage;;
-    esac
-done
-
-[ -r "$PATHS_FILE" ] || { echo "paths file $PATHS_FILE not readable" >&2; exit 2; }
-[ -r "$ENVFILE" ] || { echo "env file $ENVFILE not readable" >&2; exit 2; }
-. "$ENVFILE"
-
-exec 9>"$LOCKFILE"
-if ! flock -n 9; then
-    echo "another restic-backup run holds $LOCKFILE, aborting" >&2
-    exit 1
-fi
-
-log() { echo "[$(date '+%F %T')] $*"; }
-
-excluded() {
-    [ -r "$DB_EXCLUDE_FILE" ] && grep -qx "$1" "$DB_EXCLUDE_FILE" \
-        && log "SKIP $1 (listed in $DB_EXCLUDE_FILE)"
-}
-
-rc=0
-
-if [ "$STAGE" = all ] || [ "$STAGE" = dumps ]; then
-
-# incus's own state (instance configs, profiles, devices)
-mkdir -p "$DUMP_DIR/incus"
-incus admin sql global .dump > "$DUMP_DIR/incus/incus-global-db.sql" || rc=1
-incus admin sql local .dump > "$DUMP_DIR/incus/incus-local-db.sql" || rc=1
-
-# Database dumps, auto-discovered per running container
-PG_LIST="SELECT datname FROM pg_database WHERE NOT datistemplate AND datname <> 'postgres'"
-for ct in $(incus list status=running -c n -f csv); do
-    # native MariaDB/MySQL (unix-socket root auth), either binary naming
-    mdump=$(incus exec "$ct" -- sh -c 'command -v mariadb-dump || command -v mysqldump' 2>/dev/null)
-    mclient=$(incus exec "$ct" -- sh -c 'command -v mariadb || command -v mysql' 2>/dev/null)
-    if [ -n "$mdump" ] && [ -n "$mclient" ]; then
-        dbs=$(incus exec "$ct" -- "$mclient" -N -B -e 'SHOW DATABASES') \
-            || { echo "listing mariadb databases on $ct failed" >&2; rc=1; dbs=""; }
-        for db in $(printf '%s\n' "$dbs" \
-                    | grep -Ev '^(information_schema|performance_schema|mysql|sys)$'); do
-            excluded "$ct/$db" && continue
-            log "dump $ct/$db (mariadb)"
-            mkdir -p "$DUMP_DIR/mariadb/$ct"
-            incus exec "$ct" -- "$mdump" --single-transaction --events --routines --triggers \
-                --databases "$db" > "$DUMP_DIR/mariadb/$ct/$db.sql" \
-                || { echo "dump $ct/$db failed" >&2; rc=1; }
-        done
-        # users + grants: replayable SHOW GRANTS statements
-        log "dump $ct/grants (mariadb)"
-        incus exec "$ct" -- sh -c "$mclient -NBe \"SELECT CONCAT('SHOW GRANTS FOR ', QUOTE(user), '@', QUOTE(host), ';') FROM mysql.user\" | $mclient -NB | sed 's/\$/;/'" \
-            > "$DUMP_DIR/mariadb/$ct/grants.sql" \
-            || { echo "grants dump on $ct failed" >&2; rc=1; }
-    elif [ -n "$mdump$mclient" ]; then
-        echo "$ct has only one of dump/client mariadb binaries, skipping" >&2; rc=1
-    fi
-
-    # native PostgreSQL (peer auth as the postgres user)
-    if incus exec "$ct" -- sh -c 'command -v pg_dump' >/dev/null 2>&1; then
-        mkdir -p "$DUMP_DIR/postgres/$ct"
-        incus exec "$ct" -- su -s /bin/sh postgres -c "pg_dumpall --globals-only" \
-            > "$DUMP_DIR/postgres/$ct/globals.sql" || rc=1
-        dbs=$(incus exec "$ct" -- su -s /bin/sh postgres -c "psql -AtX -c \"$PG_LIST\"") \
-            || { echo "listing postgres databases on $ct failed" >&2; rc=1; dbs=""; }
-        for db in $dbs; do
-            excluded "$ct/$db" && continue
-            log "dump $ct/$db (postgres)"
-            incus exec "$ct" -- su -s /bin/sh postgres -c "pg_dump --clean --if-exists $db" \
-                > "$DUMP_DIR/postgres/$ct/$db.sql" \
-                || { echo "dump $ct/$db failed" >&2; rc=1; }
-        done
-    fi
-
-    # PostgreSQL inside docker (e.g. outline, login)
-    incus exec "$ct" -- sh -c 'command -v docker' >/dev/null 2>&1 || continue
-    for dc in $(incus exec "$ct" -- docker ps --format '{{.Names}} {{.Image}}' 2>/dev/null \
-                | awk 'tolower($2) ~ /postgres/ {print $1}'); do
-        pguser=$(incus exec "$ct" -- docker exec "$dc" sh -c 'echo "${POSTGRES_USER:-postgres}"') \
-            || { echo "reading POSTGRES_USER on $ct/$dc failed" >&2; rc=1; continue; }
-        mkdir -p "$DUMP_DIR/postgres/$ct/$dc"
-        incus exec "$ct" -- docker exec "$dc" pg_dumpall -U "$pguser" --globals-only \
-            > "$DUMP_DIR/postgres/$ct/$dc/globals.sql" || rc=1
-        dbs=$(incus exec "$ct" -- docker exec "$dc" psql -U "$pguser" -AtX -c "$PG_LIST") \
-            || { echo "listing postgres databases on $ct/$dc failed" >&2; rc=1; dbs=""; }
-        for db in $dbs; do
-            excluded "$ct/$dc/$db" && continue
-            log "dump $ct/$dc/$db (postgres)"
-            incus exec "$ct" -- docker exec "$dc" pg_dump -U "$pguser" --clean --if-exists "$db" \
-                > "$DUMP_DIR/postgres/$ct/$dc/$db.sql" \
-                || { echo "dump $ct/$dc/$db failed" >&2; rc=1; }
-        done
-    done
-done
-
-fi  # stage dumps
-
-if [ "$STAGE" = all ] || [ "$STAGE" = backup ]; then
-
-# clear locks left by a killed run (safe: only stale ones are removed);
-# otherwise a single interrupted backup blocks forget/prune for ever
-$RESTIC -r "$REPO" unlock >/dev/null 2>&1 || true
-
-log "restic backup -> $REPO"
-$RESTIC -r "$REPO" backup \
-    --pack-size 64 --read-concurrency 8 -o s3.connections=8 \
-    --exclude-file "$EXCLUDE_FILE" \
-    --files-from-verbatim "$PATHS_FILE" "$DUMP_DIR" \
-    || { echo "restic backup failed" >&2; rc=1; }
-
-log "forget: keep 14d/8w/6m"
-$RESTIC -r "$REPO" forget --group-by host \
-    --keep-daily 14 --keep-weekly 8 --keep-monthly 6 || rc=1
-
-fi  # stage backup
-
-log "done (rc=$rc)"
-exit $rc

restic-exclude

@@ -1,2 +0,0 @@
-/var/lib/incus/storage-pools/data/containers/nextcloud/rootfs/nextcloud/data/appdata_*/preview
-/var/lib/incus/storage-pools/data/containers/nextcloud/rootfs/nextcloud/data/appdata_*/dav-photocache

restic-incus-backup.sh

@@ -1,121 +0,0 @@
-#!/bin/bash
-#
-# Nightly restic backup of the incus instances (repo: restic-incus).
-# For every instance of the `backup` project (quiesced replicas —
-# stopped, refreshed by the 01:00 incus-copy), mount its filesystem
-# with `incus file mount` (FUSE over the per-instance sftp API, works
-# on stopped containers, needs sshfs) and back it up per-file,
-# together with its expanded config.
-#
-# ⚠️ Each instance gets its OWN mountpoint (/run/restic-incus/<name>):
-# restic selects a snapshot's parent by host+path, so a shared
-# mountpoint would parent every snapshot on the previous *other*
-# instance and force nightly full re-reads.
-#
-# Runs CHAINED after incus-copy.sh in the same cron entry — the
-# snapshot is only as fresh as the last completed replica refresh:
-#   0 1 * * * incus-copy.sh -p backup -s backup >> /var/log/incus-copy.log 2>&1 ; restic-incus-backup.sh >> /var/log/restic-incus.log 2>&1
-#
-# Usage: restic-incus-backup.sh [-r <repo>] [-p <project>]
-#          [-x <exclude,list>] [-i <only,these>]
-#
-# All instances by default; opt-out via -x (logged loudly — the list
-# cannot rot silently).
-
-set -u
-
-REPO=s3:s3.sbg.io.cloud.ovh.net/restic-incus
-PROJECT=backup
-EXCLUDE_INSTANCES="nextcloud,seafile"   # seek-bound giants: data covered by restic-data
-ONLY_INSTANCES=""
-MNT_ROOT=/run/restic-incus
-LOCKFILE=/run/lock/restic-incus-backup.lock
-ENVFILE=/root/.restic-env
-RESTIC=/usr/local/bin/restic   # cron PATH lacks /usr/local/bin
-MOUNT_TIMEOUT=30
-
-usage() {
-    echo "Usage: $0 [-r <repo>] [-p <project>] [-x <exclude,list>] [-i <only,list>]" >&2
-    exit 2
-}
-
-while getopts r:p:x:i: flag; do
-    case "${flag}" in
-        r) REPO=${OPTARG};;
-        p) PROJECT=${OPTARG};;
-        x) EXCLUDE_INSTANCES=${OPTARG};;
-        i) ONLY_INSTANCES=${OPTARG};;
-        *) usage;;
-    esac
-done
-
-[ -r "$ENVFILE" ] || { echo "env file $ENVFILE not readable" >&2; exit 2; }
-. "$ENVFILE"
-command -v sshfs >/dev/null || { echo "sshfs not installed (needed by incus file mount)" >&2; exit 2; }
-
-exec 9>"$LOCKFILE"
-if ! flock -n 9; then
-    echo "another restic-incus-backup run holds $LOCKFILE, aborting" >&2
-    exit 1
-fi
-
-log() { echo "[$(date '+%F %T')] $*"; }
-
-cleanup_mount() {  # $1 = mountpoint, $2 = mount pid
-    [ -n "${2:-}" ] && kill "$2" 2>/dev/null
-    for _ in 1 2 3 4 5; do
-        mountpoint -q "$1" || return 0
-        fusermount -u "$1" 2>/dev/null || umount "$1" 2>/dev/null
-        sleep 1
-    done
-    mountpoint -q "$1" && { echo "failed to unmount $1" >&2; return 1; }
-    return 0
-}
-
-rc=0
-mkdir -p "$MNT_ROOT"
-
-for inst in $(incus list --project "$PROJECT" -c n -f csv); do
-    if [ -n "$ONLY_INSTANCES" ]; then
-        case ",$ONLY_INSTANCES," in *",$inst,"*) ;; *) continue;; esac
-    fi
-    case ",$EXCLUDE_INSTANCES," in
-        *",$inst,"*) log "SKIP $inst (excluded)"; continue;;
-    esac
-
-    mnt="$MNT_ROOT/$inst"
-    mkdir -p "$mnt"
-    mountpoint -q "$mnt" && cleanup_mount "$mnt" ""   # stale from a killed run
-
-    # instance definition, backed up alongside the tree
-    incus config show "$inst" --project "$PROJECT" --expanded > "$MNT_ROOT/$inst.yaml" \
-        || { echo "config dump of $inst failed" >&2; rc=1; }
-
-    incus file mount "$inst/" "$mnt" --project "$PROJECT" >/dev/null 2>&1 &
-    mpid=$!
-    mounted=""
-    for _ in $(seq "$MOUNT_TIMEOUT"); do
-        mountpoint -q "$mnt" && { mounted=1; break; }
-        kill -0 "$mpid" 2>/dev/null || break
-        sleep 1
-    done
-    if [ -z "$mounted" ]; then
-        echo "mount of $inst failed" >&2; rc=1
-        cleanup_mount "$mnt" "$mpid"
-        continue
-    fi
-
-    log "backup $inst"
-    $RESTIC -r "$REPO" backup \
-        --pack-size 64 --read-concurrency 8 -o s3.connections=8 \
-        --tag "$inst" "$mnt" "$MNT_ROOT/$inst.yaml" \
-        || { echo "backup of $inst failed" >&2; rc=1; }
-
-    cleanup_mount "$mnt" "$mpid" || rc=1
-done
-
-log "forget: keep 14d/8w/6m"
-$RESTIC -r "$REPO" forget --keep-daily 14 --keep-weekly 8 --keep-monthly 6 || rc=1
-
-log "done (rc=$rc)"
-exit $rc

restic-maintenance.sh

@@ -1,54 +0,0 @@
-#!/bin/bash
-#
-# Weekly restic maintenance for both repos (Sunday, offset from the
-# nightly backups — prune takes an EXCLUSIVE lock). Per repo:
-#   - unlock: clear locks left by crashed runs
-#   - prune: --max-unused 10% (our dead-data rate is ~0.1–0.4 GiB/day
-#     vs ~1 T repos, so pruning can skip repacking for months) and
-#     --max-repack-size 4G (bounds any single Sunday's rewrite to
-#     ~10-15 min even after a mass deletion; the rest defers)
-#   - check: structure every week, plus a rotating 1/52 data subset —
-#     a full verification of every byte once a year
-#
-# Usage: restic-maintenance.sh [-r <repo>[,<repo>...]]
-
-set -u
-
-REPOS="s3:s3.sbg.io.cloud.ovh.net/restic-data,s3:s3.sbg.io.cloud.ovh.net/restic-incus"
-LOCKFILE=/run/lock/restic-maintenance.lock
-ENVFILE=/root/.restic-env
-RESTIC=/usr/local/bin/restic   # cron PATH lacks /usr/local/bin
-
-while getopts r: flag; do
-    case "${flag}" in
-        r) REPOS=${OPTARG};;
-        *) echo "Usage: $0 [-r <repo>[,<repo>...]]" >&2; exit 2;;
-    esac
-done
-
-[ -r "$ENVFILE" ] || { echo "env file $ENVFILE not readable" >&2; exit 2; }
-. "$ENVFILE"
-
-exec 9>"$LOCKFILE"
-if ! flock -n 9; then
-    echo "another restic-maintenance run holds $LOCKFILE, aborting" >&2
-    exit 1
-fi
-
-log() { echo "[$(date '+%F %T')] $*"; }
-
-# rotate the read-data subset weekly: full coverage once a year
-WEEK=$(( ($(date +%s) / 604800) % 52 + 1 ))
-
-rc=0
-for repo in $(printf '%s' "$REPOS" | tr ',' ' '); do
-    log "maintenance: $repo"
-    $RESTIC -r "$repo" unlock || rc=1
-    $RESTIC -r "$repo" prune --max-unused 10% --max-repack-size 4G --pack-size 64 || rc=1
-    $RESTIC -r "$repo" check || rc=1
-    log "check --read-data-subset=$WEEK/52"
-    $RESTIC -r "$repo" check --read-data-subset="$WEEK/52" || rc=1
-done
-
-log "done (rc=$rc)"
-exit $rc

restic-paths

@@ -1,10 +0,0 @@
-/var/lib/incus/storage-pools/data/containers/bitwarden/rootfs/opt/bitwarden
-/var/lib/incus/storage-pools/data/containers/gateway/rootfs/var/www
-/var/lib/incus/storage-pools/data/containers/git/rootfs/home/git/projects
-/var/lib/incus/storage-pools/data/containers/login/rootfs/opt/authentik
-/var/lib/incus/storage-pools/data/containers/mail/rootfs/var/vmail
-/var/lib/incus/storage-pools/data/containers/mail/rootfs/var/www
-/var/lib/incus/storage-pools/data/containers/outline/rootfs/var/lib/docker/volumes/outline_storage-data/_data
-/var/lib/incus/storage-pools/data/containers/solar/rootfs/var/www/html/solar
-/var/lib/incus/storage-pools/data/containers/nextcloud/rootfs/nextcloud
-/var/lib/incus/storage-pools/data/containers/seafile/rootfs/opt/seafile

restic-restore-test.sh

@@ -1,101 +0,0 @@
-#!/bin/bash
-#
-# Restore drill for the S3 backup leg — proves the backups are not just
-# "running" but actually restorable. Non-destructive: reads the repo,
-# writes only under a temporary directory, removes it at the end.
-#
-# Two restores, the two data kinds we back up:
-#   1. a filesystem tree   -> restored, then diffed against the live tree
-#   2. a database dump     -> restored, then sanity-checked (and with
-#                             -d, loaded into a throwaway incus container)
-#
-# Usage: restic-restore-test.sh [-r <repo>] [-t <tree-path>] [-b <db-dump>]
-#                               [-w <workdir>] [-k] [-d]
-#   -k  keep the restored files (default: clean up)
-#   -d  also load the dump into a scratch container (slower, strongest)
-#
-# Env from /root/.restic-env.
-
-set -u
-
-REPO=s3:s3.sbg.io.cloud.ovh.net/restic-data
-TREE=/var/lib/incus/storage-pools/data/containers/solar/rootfs/var/www/html/solar
-DUMP=/backup/dumps/mariadb/freshrss/freshrss.sql
-WORK=/backup/restore-test-$(date +%Y%m%d-%H%M%S)
-ENVFILE=/root/.restic-env
-RESTIC=/usr/local/bin/restic
-KEEP=0
-DBTEST=0
-
-while getopts r:t:b:w:kd flag; do
-    case "${flag}" in
-        r) REPO=${OPTARG};; t) TREE=${OPTARG};; b) DUMP=${OPTARG};;
-        w) WORK=${OPTARG};; k) KEEP=1;; d) DBTEST=1;;
-        *) echo "Usage: $0 [-r repo] [-t tree] [-b dump] [-w workdir] [-k] [-d]" >&2; exit 2;;
-    esac
-done
-
-[ -r "$ENVFILE" ] || { echo "env file $ENVFILE not readable (run as root)" >&2; exit 2; }
-. "$ENVFILE"
-log() { echo "[$(date '+%F %T')] $*"; }
-rc=0; verdict() { [ "$1" -eq 0 ] && echo "PASS  $2" || { echo "FAIL  $2"; rc=1; }; }
-
-mkdir -p "$WORK" || exit 2
-log "repo $REPO"
-$RESTIC -r "$REPO" snapshots --latest 1 --compact || { echo "cannot list snapshots" >&2; exit 1; }
-
-# ---- 1. filesystem tree ----------------------------------------------------
-log "restoring tree: $TREE"
-$RESTIC -r "$REPO" restore latest --target "$WORK/tree" --include "$TREE" >/dev/null 2>&1
-verdict $? "tree restored"
-if [ -d "$WORK/tree$TREE" ]; then
-    files=$(find "$WORK/tree$TREE" -type f | wc -l)
-    log "restored $files files; diffing against the live tree"
-    # differences are expected if the tree changed since the snapshot —
-    # report them, do not fail on them
-    if diff -qr "$WORK/tree$TREE" "$TREE" > "$WORK/diff.txt" 2>&1; then
-        verdict 0 "restored tree is identical to live ($files files)"
-    else
-        echo "NOTE  $(wc -l < "$WORK/diff.txt") path(s) differ from live (expected if changed since the snapshot):"
-        head -5 "$WORK/diff.txt" | sed 's/^/      /'
-        verdict 0 "restored tree readable ($files files)"
-    fi
-else
-    verdict 1 "restored tree missing at $WORK/tree$TREE"
-fi
-
-# ---- 2. database dump ------------------------------------------------------
-log "restoring dump: $DUMP"
-$RESTIC -r "$REPO" restore latest --target "$WORK/db" --include "$DUMP" >/dev/null 2>&1
-f="$WORK/db$DUMP"
-if [ -s "$f" ]; then
-    tables=$(grep -c "^CREATE TABLE" "$f")
-    tail -3 "$f" | grep -q "Dump completed"
-    complete=$?
-    log "dump: $(du -h "$f" | cut -f1), $tables CREATE TABLE, completed-marker=$([ $complete -eq 0 ] && echo yes || echo NO)"
-    verdict $(( complete != 0 || tables == 0 ? 1 : 0 )) "dump restored and well-formed"
-
-    if [ "$DBTEST" -eq 1 ]; then
-        ct=restore-drill-$$
-        log "loading it into a scratch container ($ct) — this takes a few minutes"
-        if incus launch images:debian/13 "$ct" >/dev/null 2>&1 &&
-           incus exec "$ct" -- sh -c "DEBIAN_FRONTEND=noninteractive apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq install -y mariadb-server" >/dev/null 2>&1; then
-            incus file push "$f" "$ct/root/dump.sql" >/dev/null 2>&1
-            incus exec "$ct" -- sh -c "mariadb < /root/dump.sql" && loaded=0 || loaded=1
-            got=$(incus exec "$ct" -- mariadb -N -B -e \
-                 "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema NOT IN ('mysql','information_schema','performance_schema','sys')" 2>/dev/null)
-            log "tables in the restored database: ${got:-0} (dump declared $tables)"
-            verdict $(( loaded != 0 || ${got:-0} == 0 ? 1 : 0 )) "dump loads into a live MariaDB"
-        else
-            verdict 1 "could not prepare the scratch container"
-        fi
-        incus delete -f "$ct" >/dev/null 2>&1
-    fi
-else
-    verdict 1 "dump restored at $f"
-fi
-
-# ---- cleanup ---------------------------------------------------------------
-if [ "$KEEP" -eq 1 ]; then log "keeping $WORK"; else rm -rf "$WORK"; log "cleaned up $WORK"; fi
-log "restore drill done (rc=$rc)"
-exit $rc

zfs-auto-snapshot.sh

@@ -20,10 +20,10 @@ TS=$(date '+%s')
 ZFS=$(which zfs)
 CMD="$ZFS snapshot ${DATASET}@snap${TS}"
 echo "[${DATE}] $CMD"
-$CMD 2>&1
+$CMD 2>&1 > /dev/null
 
 ZFS_PRUNE_SNAPSHOTS=$(which zfs-prune-snapshots)
-CMD="$ZFS_PRUNE_SNAPSHOTS -p snap $EXPIRATION $DATASET"
+CMD="$ZFS_PRUNE_SNAPSHOTS -p 'snap' $EXPIRATION $DATASET"
 echo "[${DATE}] $CMD"
-$CMD 2>&1
+$CMD 2>&1 > /dev/null
 

zpool-health.sh

@@ -1,86 +0,0 @@
-#!/bin/bash
-#
-# Pool health watchdog. ZED does not cover everything: a pool that gets
-# SUSPENDED after an I/O failure keeps its vdev marked ONLINE, so
-# statechange-notify.sh never fires, and data-class events are silent
-# unless ZED_NOTIFY_DATA is set. Three separate incidents on nuc went
-# unnoticed for days because of that.
-#
-# This checks `zpool status -x` and mails on TRANSITIONS
-# (healthy -> problem, problem -> healthy), plus a low-rate reminder
-# while a problem persists — otherwise a pool stuck unhealthy (e.g.
-# stale `<metadata>` entries in the error log) parks the watchdog in
-# the alarm state, where it can no longer signal anything NEW.
-# Quiet by default, cannot spam. Run every 15 min from cron.
-#
-# Usage: zpool-health.sh [-m <mail-to>] [-s <state-file>] [-i <hours>] [-t]
-#          -i  hours between reminders while unhealthy (default 24, 0=off)
-#          -t  send a test mail and exit (proves the path works)
-
-set -u
-
-MAILTO=root
-STATE=/var/lib/zpool-health.state
-REMIND_H=24
-TEST=0
-HOST=$(hostname -s)
-
-while getopts m:s:i:t flag; do
-    case "${flag}" in
-        m) MAILTO=${OPTARG};;
-        s) STATE=${OPTARG};;
-        i) REMIND_H=${OPTARG};;
-        t) TEST=1;;
-        *) echo "Usage: $0 [-m <mail-to>] [-s <state-file>] [-i <hours>] [-t]" >&2; exit 2;;
-    esac
-done
-
-notify() {  # subject, body
-    if command -v mail >/dev/null 2>&1; then
-        printf '%s\n' "$2" | mail -s "$1" "$MAILTO"
-    else
-        logger -t zpool-health "$1"
-        printf '%s\n' "$2" | logger -t zpool-health
-    fi
-}
-
-if [ "$TEST" -eq 1 ]; then
-    notify "[$HOST] zpool-health test" "$(zpool status -x 2>&1)"
-    echo "test notification sent to $MAILTO"; exit 0
-fi
-
-status=$(zpool status -x 2>&1)
-if [ "$status" = "all pools are healthy" ]; then
-    now=ok
-else
-    now=problem
-fi
-
-# state file: "<status> <epoch of last notification>"
-read -r was last < "$STATE" 2>/dev/null || { was=ok; last=0; }
-[ -n "${last:-}" ] || last=0
-nowsec=$(date +%s)
-
-remind=0
-if [ "$now" = problem ] && [ "$was" = problem ] && [ "$REMIND_H" -gt 0 ]; then
-    [ $(( nowsec - last )) -ge $(( REMIND_H * 3600 )) ] && remind=1
-fi
-
-if [ "$now" = "$was" ] && [ "$remind" -eq 0 ]; then
-    printf '%s %s' "$now" "$last" > "$STATE"      # keep the notify time
-    [ "$now" = problem ] && exit 1 || exit 0
-fi
-
-printf '%s %s' "$now" "$nowsec" > "$STATE"
-
-if [ "$now" = problem ]; then
-    subj="[$HOST] ZFS POOL PROBLEM"
-    [ "$remind" -eq 1 ] && subj="[$HOST] ZFS pool STILL unhealthy (${REMIND_H}h reminder)"
-    notify "$subj" "$(printf '%s\n\n%s\n' "$status" "$(zpool status -v 2>&1)")"
-    echo "$(date '+%F %T') problem: $status" >&2
-    exit 1
-else
-    notify "[$HOST] ZFS pools healthy again" "$(zpool status 2>&1)"
-    echo "$(date '+%F %T') recovered"
-fi
-exit 0