From 590279190a9dfda1b0fdb2609de77b4548113b86 Mon Sep 17 00:00:00 2001 From: Julien Lutran Date: Tue, 25 Aug 2026 12:53:16 +0200 Subject: [PATCH] Add restic drivers: data leg, incus leg via file mount, weekly maintenance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit restic-backup.sh: dump phase carried over verbatim from plakar-backup.sh, then one restic backup invocation (dump dir + restic-paths, nextcloud excludes) into restic-data; -s dumps|backup for staged seeding. restic-incus-backup.sh: per-instance incus file mount (distinct mountpoints — restic parents by host+path), expanded config yaml alongside, giants excluded. restic-maintenance.sh: weekly prune (--max-unused 10%, --max-repack-size 4G) + structure check + rotating 1/52 data check. Plus restic-paths and the nextcloud excludes list. Co-Authored-By: Claude Fable 5 --- restic-backup.sh | 160 +++++++++++++++++++++++++++++++++++++++ restic-incus-backup.sh | 120 +++++++++++++++++++++++++++++ restic-maintenance.sh | 53 +++++++++++++ restic-nextcloud-exclude | 2 + restic-paths | 11 +++ 5 files changed, 346 insertions(+) create mode 100755 restic-backup.sh create mode 100755 restic-incus-backup.sh create mode 100755 restic-maintenance.sh create mode 100644 restic-nextcloud-exclude create mode 100644 restic-paths diff --git a/restic-backup.sh b/restic-backup.sh new file mode 100755 index 0000000..5fa3883 --- /dev/null +++ b/restic-backup.sh @@ -0,0 +1,160 @@ +#!/bin/bash +# +# Nightly restic data backup (repo: restic-data). Three phases: +# 1. dumps — incus's own DBs, plus application-consistent database +# dumps taken with `incus exec` (container-local auth: no DB +# users, no network exposure). Auto-discovered in every RUNNING +# container — no manifest to go stale: native MariaDB/MySQL +# (either binary naming), native PostgreSQL (pg_dump per DB + +# pg_dumpall --globals-only), and PostgreSQL inside docker +# containers (image name matching "postgres"). Dumps staged +# plain (not gzipped — CDC dedup needs uncompressed input). +# 2. one `restic backup` invocation: the dump dir + every path in +# the paths file (one index load, one snapshot per night); +# exclude patterns applied globally. +# 3. retention — `restic forget --group-by host` (seed-era +# snapshots have different path sets and must age in one group). +# Prune/check live in restic-maintenance.sh (weekly). +# +# Usage: restic-backup.sh [-r ] [-f ] [-d ] +# [-s ] # stage: dumps|backup|all (default all); +# # -s backup skips re-dumping (seeding aid) +# +# Env from /root/.restic-env (AWS creds, RESTIC_PASSWORD_FILE, +# RESTIC_CACHE_DIR). + +set -u + +REPO=s3:s3.sbg.io.cloud.ovh.net/restic-data +PATHS_FILE=/root/scripts/restic-paths +EXCLUDE_FILE=/root/scripts/restic-nextcloud-exclude +DB_EXCLUDE_FILE=/root/scripts/plakar-db-exclude # same opt-out list, same format +DUMP_DIR=/backup/dumps +LOCKFILE=/run/lock/restic-backup.lock +ENVFILE=/root/.restic-env +STAGE=all + +usage() { + echo "Usage: $0 [-r ] [-f ] [-d ] [-s dumps|backup|all]" >&2 + exit 2 +} + +while getopts r:f:d:s: flag; do + case "${flag}" in + r) REPO=${OPTARG};; + f) PATHS_FILE=${OPTARG};; + d) DUMP_DIR=${OPTARG};; + s) STAGE=${OPTARG};; + *) usage;; + esac +done + +[ -r "$PATHS_FILE" ] || { echo "paths file $PATHS_FILE not readable" >&2; exit 2; } +[ -r "$ENVFILE" ] || { echo "env file $ENVFILE not readable" >&2; exit 2; } +. "$ENVFILE" + +exec 9>"$LOCKFILE" +if ! flock -n 9; then + echo "another restic-backup run holds $LOCKFILE, aborting" >&2 + exit 1 +fi + +log() { echo "[$(date '+%F %T')] $*"; } + +excluded() { + [ -r "$DB_EXCLUDE_FILE" ] && grep -qx "$1" "$DB_EXCLUDE_FILE" \ + && log "SKIP $1 (listed in $DB_EXCLUDE_FILE)" +} + +rc=0 + +if [ "$STAGE" = all ] || [ "$STAGE" = dumps ]; then + +# incus's own state (instance configs, profiles, devices) +mkdir -p "$DUMP_DIR/incus" +incus admin sql global .dump > "$DUMP_DIR/incus/incus-global-db.sql" || rc=1 +incus admin sql local .dump > "$DUMP_DIR/incus/incus-local-db.sql" || rc=1 + +# Database dumps, auto-discovered per running container +PG_LIST="SELECT datname FROM pg_database WHERE NOT datistemplate AND datname <> 'postgres'" +for ct in $(incus list status=running -c n -f csv); do + # native MariaDB/MySQL (unix-socket root auth), either binary naming + mdump=$(incus exec "$ct" -- sh -c 'command -v mariadb-dump || command -v mysqldump' 2>/dev/null) + mclient=$(incus exec "$ct" -- sh -c 'command -v mariadb || command -v mysql' 2>/dev/null) + if [ -n "$mdump" ] && [ -n "$mclient" ]; then + dbs=$(incus exec "$ct" -- "$mclient" -N -B -e 'SHOW DATABASES') \ + || { echo "listing mariadb databases on $ct failed" >&2; rc=1; dbs=""; } + for db in $(printf '%s\n' "$dbs" \ + | grep -Ev '^(information_schema|performance_schema|mysql|sys)$'); do + excluded "$ct/$db" && continue + log "dump $ct/$db (mariadb)" + mkdir -p "$DUMP_DIR/mariadb/$ct" + incus exec "$ct" -- "$mdump" --single-transaction --events --routines --triggers \ + --databases "$db" > "$DUMP_DIR/mariadb/$ct/$db.sql" \ + || { echo "dump $ct/$db failed" >&2; rc=1; } + done + # users + grants: replayable SHOW GRANTS statements + log "dump $ct/grants (mariadb)" + incus exec "$ct" -- sh -c "$mclient -NBe \"SELECT CONCAT('SHOW GRANTS FOR ', QUOTE(user), '@', QUOTE(host), ';') FROM mysql.user\" | $mclient -NB | sed 's/\$/;/'" \ + > "$DUMP_DIR/mariadb/$ct/grants.sql" \ + || { echo "grants dump on $ct failed" >&2; rc=1; } + elif [ -n "$mdump$mclient" ]; then + echo "$ct has only one of dump/client mariadb binaries, skipping" >&2; rc=1 + fi + + # native PostgreSQL (peer auth as the postgres user) + if incus exec "$ct" -- sh -c 'command -v pg_dump' >/dev/null 2>&1; then + mkdir -p "$DUMP_DIR/postgres/$ct" + incus exec "$ct" -- su -s /bin/sh postgres -c "pg_dumpall --globals-only" \ + > "$DUMP_DIR/postgres/$ct/globals.sql" || rc=1 + dbs=$(incus exec "$ct" -- su -s /bin/sh postgres -c "psql -AtX -c \"$PG_LIST\"") \ + || { echo "listing postgres databases on $ct failed" >&2; rc=1; dbs=""; } + for db in $dbs; do + excluded "$ct/$db" && continue + log "dump $ct/$db (postgres)" + incus exec "$ct" -- su -s /bin/sh postgres -c "pg_dump --clean --if-exists $db" \ + > "$DUMP_DIR/postgres/$ct/$db.sql" \ + || { echo "dump $ct/$db failed" >&2; rc=1; } + done + fi + + # PostgreSQL inside docker (e.g. outline, login) + incus exec "$ct" -- sh -c 'command -v docker' >/dev/null 2>&1 || continue + for dc in $(incus exec "$ct" -- docker ps --format '{{.Names}} {{.Image}}' 2>/dev/null \ + | awk 'tolower($2) ~ /postgres/ {print $1}'); do + pguser=$(incus exec "$ct" -- docker exec "$dc" sh -c 'echo "${POSTGRES_USER:-postgres}"') \ + || { echo "reading POSTGRES_USER on $ct/$dc failed" >&2; rc=1; continue; } + mkdir -p "$DUMP_DIR/postgres/$ct/$dc" + incus exec "$ct" -- docker exec "$dc" pg_dumpall -U "$pguser" --globals-only \ + > "$DUMP_DIR/postgres/$ct/$dc/globals.sql" || rc=1 + dbs=$(incus exec "$ct" -- docker exec "$dc" psql -U "$pguser" -AtX -c "$PG_LIST") \ + || { echo "listing postgres databases on $ct/$dc failed" >&2; rc=1; dbs=""; } + for db in $dbs; do + excluded "$ct/$dc/$db" && continue + log "dump $ct/$dc/$db (postgres)" + incus exec "$ct" -- docker exec "$dc" pg_dump -U "$pguser" --clean --if-exists "$db" \ + > "$DUMP_DIR/postgres/$ct/$dc/$db.sql" \ + || { echo "dump $ct/$dc/$db failed" >&2; rc=1; } + done + done +done + +fi # stage dumps + +if [ "$STAGE" = all ] || [ "$STAGE" = backup ]; then + +log "restic backup -> $REPO" +restic -r "$REPO" backup \ + --pack-size 64 --read-concurrency 8 -o s3.connections=8 \ + --exclude-file "$EXCLUDE_FILE" \ + --files-from-verbatim "$PATHS_FILE" "$DUMP_DIR" \ + || { echo "restic backup failed" >&2; rc=1; } + +log "forget: keep 14d/8w/6m" +restic -r "$REPO" forget --group-by host \ + --keep-daily 14 --keep-weekly 8 --keep-monthly 6 || rc=1 + +fi # stage backup + +log "done (rc=$rc)" +exit $rc diff --git a/restic-incus-backup.sh b/restic-incus-backup.sh new file mode 100755 index 0000000..d7ddb23 --- /dev/null +++ b/restic-incus-backup.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# +# Nightly restic backup of the incus instances (repo: restic-incus). +# For every instance of the `backup` project (quiesced replicas — +# stopped, refreshed by the 01:00 incus-copy), mount its filesystem +# with `incus file mount` (FUSE over the per-instance sftp API, works +# on stopped containers, needs sshfs) and back it up per-file, +# together with its expanded config. +# +# ⚠️ Each instance gets its OWN mountpoint (/run/restic-incus/): +# restic selects a snapshot's parent by host+path, so a shared +# mountpoint would parent every snapshot on the previous *other* +# instance and force nightly full re-reads. +# +# Runs CHAINED after incus-copy.sh in the same cron entry — the +# snapshot is only as fresh as the last completed replica refresh: +# 0 1 * * * incus-copy.sh -p backup -s backup >> /var/log/incus-copy.log 2>&1 ; restic-incus-backup.sh >> /var/log/restic-incus.log 2>&1 +# +# Usage: restic-incus-backup.sh [-r ] [-p ] +# [-x ] [-i ] +# +# All instances by default; opt-out via -x (logged loudly — the list +# cannot rot silently). + +set -u + +REPO=s3:s3.sbg.io.cloud.ovh.net/restic-incus +PROJECT=backup +EXCLUDE_INSTANCES="nextcloud,seafile" # seek-bound giants: data covered by restic-data +ONLY_INSTANCES="" +MNT_ROOT=/run/restic-incus +LOCKFILE=/run/lock/restic-incus-backup.lock +ENVFILE=/root/.restic-env +MOUNT_TIMEOUT=30 + +usage() { + echo "Usage: $0 [-r ] [-p ] [-x ] [-i ]" >&2 + exit 2 +} + +while getopts r:p:x:i: flag; do + case "${flag}" in + r) REPO=${OPTARG};; + p) PROJECT=${OPTARG};; + x) EXCLUDE_INSTANCES=${OPTARG};; + i) ONLY_INSTANCES=${OPTARG};; + *) usage;; + esac +done + +[ -r "$ENVFILE" ] || { echo "env file $ENVFILE not readable" >&2; exit 2; } +. "$ENVFILE" +command -v sshfs >/dev/null || { echo "sshfs not installed (needed by incus file mount)" >&2; exit 2; } + +exec 9>"$LOCKFILE" +if ! flock -n 9; then + echo "another restic-incus-backup run holds $LOCKFILE, aborting" >&2 + exit 1 +fi + +log() { echo "[$(date '+%F %T')] $*"; } + +cleanup_mount() { # $1 = mountpoint, $2 = mount pid + [ -n "${2:-}" ] && kill "$2" 2>/dev/null + for _ in 1 2 3 4 5; do + mountpoint -q "$1" || return 0 + fusermount -u "$1" 2>/dev/null || umount "$1" 2>/dev/null + sleep 1 + done + mountpoint -q "$1" && { echo "failed to unmount $1" >&2; return 1; } + return 0 +} + +rc=0 +mkdir -p "$MNT_ROOT" + +for inst in $(incus list --project "$PROJECT" -c n -f csv); do + if [ -n "$ONLY_INSTANCES" ]; then + case ",$ONLY_INSTANCES," in *",$inst,"*) ;; *) continue;; esac + fi + case ",$EXCLUDE_INSTANCES," in + *",$inst,"*) log "SKIP $inst (excluded)"; continue;; + esac + + mnt="$MNT_ROOT/$inst" + mkdir -p "$mnt" + mountpoint -q "$mnt" && cleanup_mount "$mnt" "" # stale from a killed run + + # instance definition, backed up alongside the tree + incus config show "$inst" --project "$PROJECT" --expanded > "$MNT_ROOT/$inst.yaml" \ + || { echo "config dump of $inst failed" >&2; rc=1; } + + incus file mount "$inst/" "$mnt" --project "$PROJECT" >/dev/null 2>&1 & + mpid=$! + mounted="" + for _ in $(seq "$MOUNT_TIMEOUT"); do + mountpoint -q "$mnt" && { mounted=1; break; } + kill -0 "$mpid" 2>/dev/null || break + sleep 1 + done + if [ -z "$mounted" ]; then + echo "mount of $inst failed" >&2; rc=1 + cleanup_mount "$mnt" "$mpid" + continue + fi + + log "backup $inst" + restic -r "$REPO" backup \ + --pack-size 64 --read-concurrency 8 -o s3.connections=8 \ + --tag "$inst" "$mnt" "$MNT_ROOT/$inst.yaml" \ + || { echo "backup of $inst failed" >&2; rc=1; } + + cleanup_mount "$mnt" "$mpid" || rc=1 +done + +log "forget: keep 14d/8w/6m" +restic -r "$REPO" forget --keep-daily 14 --keep-weekly 8 --keep-monthly 6 || rc=1 + +log "done (rc=$rc)" +exit $rc diff --git a/restic-maintenance.sh b/restic-maintenance.sh new file mode 100755 index 0000000..46021ea --- /dev/null +++ b/restic-maintenance.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# +# Weekly restic maintenance for both repos (Sunday, offset from the +# nightly backups — prune takes an EXCLUSIVE lock). Per repo: +# - unlock: clear locks left by crashed runs +# - prune: --max-unused 10% (our dead-data rate is ~0.1–0.4 GiB/day +# vs ~1 T repos, so pruning can skip repacking for months) and +# --max-repack-size 4G (bounds any single Sunday's rewrite to +# ~10-15 min even after a mass deletion; the rest defers) +# - check: structure every week, plus a rotating 1/52 data subset — +# a full verification of every byte once a year +# +# Usage: restic-maintenance.sh [-r [,...]] + +set -u + +REPOS="s3:s3.sbg.io.cloud.ovh.net/restic-data,s3:s3.sbg.io.cloud.ovh.net/restic-incus" +LOCKFILE=/run/lock/restic-maintenance.lock +ENVFILE=/root/.restic-env + +while getopts r: flag; do + case "${flag}" in + r) REPOS=${OPTARG};; + *) echo "Usage: $0 [-r [,...]]" >&2; exit 2;; + esac +done + +[ -r "$ENVFILE" ] || { echo "env file $ENVFILE not readable" >&2; exit 2; } +. "$ENVFILE" + +exec 9>"$LOCKFILE" +if ! flock -n 9; then + echo "another restic-maintenance run holds $LOCKFILE, aborting" >&2 + exit 1 +fi + +log() { echo "[$(date '+%F %T')] $*"; } + +# rotate the read-data subset weekly: full coverage once a year +WEEK=$(( ($(date +%s) / 604800) % 52 + 1 )) + +rc=0 +for repo in $(printf '%s' "$REPOS" | tr ',' ' '); do + log "maintenance: $repo" + restic -r "$repo" unlock || rc=1 + restic -r "$repo" prune --max-unused 10% --max-repack-size 4G --pack-size 64 || rc=1 + restic -r "$repo" check || rc=1 + log "check --read-data-subset=$WEEK/52" + restic -r "$repo" check --read-data-subset="$WEEK/52" || rc=1 +done + +log "done (rc=$rc)" +exit $rc diff --git a/restic-nextcloud-exclude b/restic-nextcloud-exclude new file mode 100644 index 0000000..138dba9 --- /dev/null +++ b/restic-nextcloud-exclude @@ -0,0 +1,2 @@ +/var/lib/incus/storage-pools/data/containers/nextcloud/rootfs/nextcloud/data/appdata_*/preview +/var/lib/incus/storage-pools/data/containers/nextcloud/rootfs/nextcloud/data/appdata_*/dav-photocache diff --git a/restic-paths b/restic-paths new file mode 100644 index 0000000..6f1091e --- /dev/null +++ b/restic-paths @@ -0,0 +1,11 @@ +# /root/scripts/restic-paths — one path per line (--files-from-verbatim) +/var/lib/incus/storage-pools/data/containers/bitwarden/rootfs/opt/bitwarden +/var/lib/incus/storage-pools/data/containers/gateway/rootfs/var/www +/var/lib/incus/storage-pools/data/containers/git/rootfs/home/git/projects +/var/lib/incus/storage-pools/data/containers/login/rootfs/opt/authentik +/var/lib/incus/storage-pools/data/containers/mail/rootfs/var/vmail +/var/lib/incus/storage-pools/data/containers/mail/rootfs/var/www +/var/lib/incus/storage-pools/data/containers/outline/rootfs/var/lib/docker/volumes/outline_storage-data/_data +/var/lib/incus/storage-pools/data/containers/solar/rootfs/var/www/html/solar +/var/lib/incus/storage-pools/data/containers/nextcloud/rootfs/nextcloud +/var/lib/incus/storage-pools/data/containers/seafile/rootfs/opt/seafile