Add restic drivers: data leg, incus leg via file mount, weekly maintenance
restic-backup.sh: dump phase carried over verbatim from plakar-backup.sh, then one restic backup invocation (dump dir + restic-paths, nextcloud excludes) into restic-data; -s dumps|backup for staged seeding. restic-incus-backup.sh: per-instance incus file mount (distinct mountpoints — restic parents by host+path), expanded config yaml alongside, giants excluded. restic-maintenance.sh: weekly prune (--max-unused 10%, --max-repack-size 4G) + structure check + rotating 1/52 data check. Plus restic-paths and the nextcloud excludes list. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Julien Lutran
co-authored by
Claude Fable 5
Claude Fable 5
parent
528dec1121
commit
590279190a
Executable
+160
@@ -0,0 +1,160 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
# Nightly restic data backup (repo: restic-data). Three phases:
|
||||||
|
# 1. dumps — incus's own DBs, plus application-consistent database
|
||||||
|
# dumps taken with `incus exec` (container-local auth: no DB
|
||||||
|
# users, no network exposure). Auto-discovered in every RUNNING
|
||||||
|
# container — no manifest to go stale: native MariaDB/MySQL
|
||||||
|
# (either binary naming), native PostgreSQL (pg_dump per DB +
|
||||||
|
# pg_dumpall --globals-only), and PostgreSQL inside docker
|
||||||
|
# containers (image name matching "postgres"). Dumps staged
|
||||||
|
# plain (not gzipped — CDC dedup needs uncompressed input).
|
||||||
|
# 2. one `restic backup` invocation: the dump dir + every path in
|
||||||
|
# the paths file (one index load, one snapshot per night);
|
||||||
|
# exclude patterns applied globally.
|
||||||
|
# 3. retention — `restic forget --group-by host` (seed-era
|
||||||
|
# snapshots have different path sets and must age in one group).
|
||||||
|
# Prune/check live in restic-maintenance.sh (weekly).
|
||||||
|
#
|
||||||
|
# Usage: restic-backup.sh [-r <repo>] [-f <paths-file>] [-d <dump-dir>]
|
||||||
|
# [-s <stage>] # stage: dumps|backup|all (default all);
|
||||||
|
# # -s backup skips re-dumping (seeding aid)
|
||||||
|
#
|
||||||
|
# Env from /root/.restic-env (AWS creds, RESTIC_PASSWORD_FILE,
|
||||||
|
# RESTIC_CACHE_DIR).
|
||||||
|
|
||||||
|
set -u
|
||||||
|
|
||||||
|
REPO=s3:s3.sbg.io.cloud.ovh.net/restic-data
|
||||||
|
PATHS_FILE=/root/scripts/restic-paths
|
||||||
|
EXCLUDE_FILE=/root/scripts/restic-nextcloud-exclude
|
||||||
|
DB_EXCLUDE_FILE=/root/scripts/plakar-db-exclude # same opt-out list, same format
|
||||||
|
DUMP_DIR=/backup/dumps
|
||||||
|
LOCKFILE=/run/lock/restic-backup.lock
|
||||||
|
ENVFILE=/root/.restic-env
|
||||||
|
STAGE=all
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
echo "Usage: $0 [-r <repo>] [-f <paths-file>] [-d <dump-dir>] [-s dumps|backup|all]" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
|
while getopts r:f:d:s: flag; do
|
||||||
|
case "${flag}" in
|
||||||
|
r) REPO=${OPTARG};;
|
||||||
|
f) PATHS_FILE=${OPTARG};;
|
||||||
|
d) DUMP_DIR=${OPTARG};;
|
||||||
|
s) STAGE=${OPTARG};;
|
||||||
|
*) usage;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
[ -r "$PATHS_FILE" ] || { echo "paths file $PATHS_FILE not readable" >&2; exit 2; }
|
||||||
|
[ -r "$ENVFILE" ] || { echo "env file $ENVFILE not readable" >&2; exit 2; }
|
||||||
|
. "$ENVFILE"
|
||||||
|
|
||||||
|
exec 9>"$LOCKFILE"
|
||||||
|
if ! flock -n 9; then
|
||||||
|
echo "another restic-backup run holds $LOCKFILE, aborting" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
log() { echo "[$(date '+%F %T')] $*"; }
|
||||||
|
|
||||||
|
excluded() {
|
||||||
|
[ -r "$DB_EXCLUDE_FILE" ] && grep -qx "$1" "$DB_EXCLUDE_FILE" \
|
||||||
|
&& log "SKIP $1 (listed in $DB_EXCLUDE_FILE)"
|
||||||
|
}
|
||||||
|
|
||||||
|
rc=0
|
||||||
|
|
||||||
|
if [ "$STAGE" = all ] || [ "$STAGE" = dumps ]; then
|
||||||
|
|
||||||
|
# incus's own state (instance configs, profiles, devices)
|
||||||
|
mkdir -p "$DUMP_DIR/incus"
|
||||||
|
incus admin sql global .dump > "$DUMP_DIR/incus/incus-global-db.sql" || rc=1
|
||||||
|
incus admin sql local .dump > "$DUMP_DIR/incus/incus-local-db.sql" || rc=1
|
||||||
|
|
||||||
|
# Database dumps, auto-discovered per running container
|
||||||
|
PG_LIST="SELECT datname FROM pg_database WHERE NOT datistemplate AND datname <> 'postgres'"
|
||||||
|
for ct in $(incus list status=running -c n -f csv); do
|
||||||
|
# native MariaDB/MySQL (unix-socket root auth), either binary naming
|
||||||
|
mdump=$(incus exec "$ct" -- sh -c 'command -v mariadb-dump || command -v mysqldump' 2>/dev/null)
|
||||||
|
mclient=$(incus exec "$ct" -- sh -c 'command -v mariadb || command -v mysql' 2>/dev/null)
|
||||||
|
if [ -n "$mdump" ] && [ -n "$mclient" ]; then
|
||||||
|
dbs=$(incus exec "$ct" -- "$mclient" -N -B -e 'SHOW DATABASES') \
|
||||||
|
|| { echo "listing mariadb databases on $ct failed" >&2; rc=1; dbs=""; }
|
||||||
|
for db in $(printf '%s\n' "$dbs" \
|
||||||
|
| grep -Ev '^(information_schema|performance_schema|mysql|sys)$'); do
|
||||||
|
excluded "$ct/$db" && continue
|
||||||
|
log "dump $ct/$db (mariadb)"
|
||||||
|
mkdir -p "$DUMP_DIR/mariadb/$ct"
|
||||||
|
incus exec "$ct" -- "$mdump" --single-transaction --events --routines --triggers \
|
||||||
|
--databases "$db" > "$DUMP_DIR/mariadb/$ct/$db.sql" \
|
||||||
|
|| { echo "dump $ct/$db failed" >&2; rc=1; }
|
||||||
|
done
|
||||||
|
# users + grants: replayable SHOW GRANTS statements
|
||||||
|
log "dump $ct/grants (mariadb)"
|
||||||
|
incus exec "$ct" -- sh -c "$mclient -NBe \"SELECT CONCAT('SHOW GRANTS FOR ', QUOTE(user), '@', QUOTE(host), ';') FROM mysql.user\" | $mclient -NB | sed 's/\$/;/'" \
|
||||||
|
> "$DUMP_DIR/mariadb/$ct/grants.sql" \
|
||||||
|
|| { echo "grants dump on $ct failed" >&2; rc=1; }
|
||||||
|
elif [ -n "$mdump$mclient" ]; then
|
||||||
|
echo "$ct has only one of dump/client mariadb binaries, skipping" >&2; rc=1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# native PostgreSQL (peer auth as the postgres user)
|
||||||
|
if incus exec "$ct" -- sh -c 'command -v pg_dump' >/dev/null 2>&1; then
|
||||||
|
mkdir -p "$DUMP_DIR/postgres/$ct"
|
||||||
|
incus exec "$ct" -- su -s /bin/sh postgres -c "pg_dumpall --globals-only" \
|
||||||
|
> "$DUMP_DIR/postgres/$ct/globals.sql" || rc=1
|
||||||
|
dbs=$(incus exec "$ct" -- su -s /bin/sh postgres -c "psql -AtX -c \"$PG_LIST\"") \
|
||||||
|
|| { echo "listing postgres databases on $ct failed" >&2; rc=1; dbs=""; }
|
||||||
|
for db in $dbs; do
|
||||||
|
excluded "$ct/$db" && continue
|
||||||
|
log "dump $ct/$db (postgres)"
|
||||||
|
incus exec "$ct" -- su -s /bin/sh postgres -c "pg_dump --clean --if-exists $db" \
|
||||||
|
> "$DUMP_DIR/postgres/$ct/$db.sql" \
|
||||||
|
|| { echo "dump $ct/$db failed" >&2; rc=1; }
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
# PostgreSQL inside docker (e.g. outline, login)
|
||||||
|
incus exec "$ct" -- sh -c 'command -v docker' >/dev/null 2>&1 || continue
|
||||||
|
for dc in $(incus exec "$ct" -- docker ps --format '{{.Names}} {{.Image}}' 2>/dev/null \
|
||||||
|
| awk 'tolower($2) ~ /postgres/ {print $1}'); do
|
||||||
|
pguser=$(incus exec "$ct" -- docker exec "$dc" sh -c 'echo "${POSTGRES_USER:-postgres}"') \
|
||||||
|
|| { echo "reading POSTGRES_USER on $ct/$dc failed" >&2; rc=1; continue; }
|
||||||
|
mkdir -p "$DUMP_DIR/postgres/$ct/$dc"
|
||||||
|
incus exec "$ct" -- docker exec "$dc" pg_dumpall -U "$pguser" --globals-only \
|
||||||
|
> "$DUMP_DIR/postgres/$ct/$dc/globals.sql" || rc=1
|
||||||
|
dbs=$(incus exec "$ct" -- docker exec "$dc" psql -U "$pguser" -AtX -c "$PG_LIST") \
|
||||||
|
|| { echo "listing postgres databases on $ct/$dc failed" >&2; rc=1; dbs=""; }
|
||||||
|
for db in $dbs; do
|
||||||
|
excluded "$ct/$dc/$db" && continue
|
||||||
|
log "dump $ct/$dc/$db (postgres)"
|
||||||
|
incus exec "$ct" -- docker exec "$dc" pg_dump -U "$pguser" --clean --if-exists "$db" \
|
||||||
|
> "$DUMP_DIR/postgres/$ct/$dc/$db.sql" \
|
||||||
|
|| { echo "dump $ct/$dc/$db failed" >&2; rc=1; }
|
||||||
|
done
|
||||||
|
done
|
||||||
|
done
|
||||||
|
|
||||||
|
fi # stage dumps
|
||||||
|
|
||||||
|
if [ "$STAGE" = all ] || [ "$STAGE" = backup ]; then
|
||||||
|
|
||||||
|
log "restic backup -> $REPO"
|
||||||
|
restic -r "$REPO" backup \
|
||||||
|
--pack-size 64 --read-concurrency 8 -o s3.connections=8 \
|
||||||
|
--exclude-file "$EXCLUDE_FILE" \
|
||||||
|
--files-from-verbatim "$PATHS_FILE" "$DUMP_DIR" \
|
||||||
|
|| { echo "restic backup failed" >&2; rc=1; }
|
||||||
|
|
||||||
|
log "forget: keep 14d/8w/6m"
|
||||||
|
restic -r "$REPO" forget --group-by host \
|
||||||
|
--keep-daily 14 --keep-weekly 8 --keep-monthly 6 || rc=1
|
||||||
|
|
||||||
|
fi # stage backup
|
||||||
|
|
||||||
|
log "done (rc=$rc)"
|
||||||
|
exit $rc
|
||||||
Executable
+120
@@ -0,0 +1,120 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
# Nightly restic backup of the incus instances (repo: restic-incus).
|
||||||
|
# For every instance of the `backup` project (quiesced replicas —
|
||||||
|
# stopped, refreshed by the 01:00 incus-copy), mount its filesystem
|
||||||
|
# with `incus file mount` (FUSE over the per-instance sftp API, works
|
||||||
|
# on stopped containers, needs sshfs) and back it up per-file,
|
||||||
|
# together with its expanded config.
|
||||||
|
#
|
||||||
|
# ⚠️ Each instance gets its OWN mountpoint (/run/restic-incus/<name>):
|
||||||
|
# restic selects a snapshot's parent by host+path, so a shared
|
||||||
|
# mountpoint would parent every snapshot on the previous *other*
|
||||||
|
# instance and force nightly full re-reads.
|
||||||
|
#
|
||||||
|
# Runs CHAINED after incus-copy.sh in the same cron entry — the
|
||||||
|
# snapshot is only as fresh as the last completed replica refresh:
|
||||||
|
# 0 1 * * * incus-copy.sh -p backup -s backup >> /var/log/incus-copy.log 2>&1 ; restic-incus-backup.sh >> /var/log/restic-incus.log 2>&1
|
||||||
|
#
|
||||||
|
# Usage: restic-incus-backup.sh [-r <repo>] [-p <project>]
|
||||||
|
# [-x <exclude,list>] [-i <only,these>]
|
||||||
|
#
|
||||||
|
# All instances by default; opt-out via -x (logged loudly — the list
|
||||||
|
# cannot rot silently).
|
||||||
|
|
||||||
|
set -u
|
||||||
|
|
||||||
|
REPO=s3:s3.sbg.io.cloud.ovh.net/restic-incus
|
||||||
|
PROJECT=backup
|
||||||
|
EXCLUDE_INSTANCES="nextcloud,seafile" # seek-bound giants: data covered by restic-data
|
||||||
|
ONLY_INSTANCES=""
|
||||||
|
MNT_ROOT=/run/restic-incus
|
||||||
|
LOCKFILE=/run/lock/restic-incus-backup.lock
|
||||||
|
ENVFILE=/root/.restic-env
|
||||||
|
MOUNT_TIMEOUT=30
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
echo "Usage: $0 [-r <repo>] [-p <project>] [-x <exclude,list>] [-i <only,list>]" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
|
while getopts r:p:x:i: flag; do
|
||||||
|
case "${flag}" in
|
||||||
|
r) REPO=${OPTARG};;
|
||||||
|
p) PROJECT=${OPTARG};;
|
||||||
|
x) EXCLUDE_INSTANCES=${OPTARG};;
|
||||||
|
i) ONLY_INSTANCES=${OPTARG};;
|
||||||
|
*) usage;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
[ -r "$ENVFILE" ] || { echo "env file $ENVFILE not readable" >&2; exit 2; }
|
||||||
|
. "$ENVFILE"
|
||||||
|
command -v sshfs >/dev/null || { echo "sshfs not installed (needed by incus file mount)" >&2; exit 2; }
|
||||||
|
|
||||||
|
exec 9>"$LOCKFILE"
|
||||||
|
if ! flock -n 9; then
|
||||||
|
echo "another restic-incus-backup run holds $LOCKFILE, aborting" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
log() { echo "[$(date '+%F %T')] $*"; }
|
||||||
|
|
||||||
|
cleanup_mount() { # $1 = mountpoint, $2 = mount pid
|
||||||
|
[ -n "${2:-}" ] && kill "$2" 2>/dev/null
|
||||||
|
for _ in 1 2 3 4 5; do
|
||||||
|
mountpoint -q "$1" || return 0
|
||||||
|
fusermount -u "$1" 2>/dev/null || umount "$1" 2>/dev/null
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
mountpoint -q "$1" && { echo "failed to unmount $1" >&2; return 1; }
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
rc=0
|
||||||
|
mkdir -p "$MNT_ROOT"
|
||||||
|
|
||||||
|
for inst in $(incus list --project "$PROJECT" -c n -f csv); do
|
||||||
|
if [ -n "$ONLY_INSTANCES" ]; then
|
||||||
|
case ",$ONLY_INSTANCES," in *",$inst,"*) ;; *) continue;; esac
|
||||||
|
fi
|
||||||
|
case ",$EXCLUDE_INSTANCES," in
|
||||||
|
*",$inst,"*) log "SKIP $inst (excluded)"; continue;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
mnt="$MNT_ROOT/$inst"
|
||||||
|
mkdir -p "$mnt"
|
||||||
|
mountpoint -q "$mnt" && cleanup_mount "$mnt" "" # stale from a killed run
|
||||||
|
|
||||||
|
# instance definition, backed up alongside the tree
|
||||||
|
incus config show "$inst" --project "$PROJECT" --expanded > "$MNT_ROOT/$inst.yaml" \
|
||||||
|
|| { echo "config dump of $inst failed" >&2; rc=1; }
|
||||||
|
|
||||||
|
incus file mount "$inst/" "$mnt" --project "$PROJECT" >/dev/null 2>&1 &
|
||||||
|
mpid=$!
|
||||||
|
mounted=""
|
||||||
|
for _ in $(seq "$MOUNT_TIMEOUT"); do
|
||||||
|
mountpoint -q "$mnt" && { mounted=1; break; }
|
||||||
|
kill -0 "$mpid" 2>/dev/null || break
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
if [ -z "$mounted" ]; then
|
||||||
|
echo "mount of $inst failed" >&2; rc=1
|
||||||
|
cleanup_mount "$mnt" "$mpid"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "backup $inst"
|
||||||
|
restic -r "$REPO" backup \
|
||||||
|
--pack-size 64 --read-concurrency 8 -o s3.connections=8 \
|
||||||
|
--tag "$inst" "$mnt" "$MNT_ROOT/$inst.yaml" \
|
||||||
|
|| { echo "backup of $inst failed" >&2; rc=1; }
|
||||||
|
|
||||||
|
cleanup_mount "$mnt" "$mpid" || rc=1
|
||||||
|
done
|
||||||
|
|
||||||
|
log "forget: keep 14d/8w/6m"
|
||||||
|
restic -r "$REPO" forget --keep-daily 14 --keep-weekly 8 --keep-monthly 6 || rc=1
|
||||||
|
|
||||||
|
log "done (rc=$rc)"
|
||||||
|
exit $rc
|
||||||
Executable
+53
@@ -0,0 +1,53 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
# Weekly restic maintenance for both repos (Sunday, offset from the
|
||||||
|
# nightly backups — prune takes an EXCLUSIVE lock). Per repo:
|
||||||
|
# - unlock: clear locks left by crashed runs
|
||||||
|
# - prune: --max-unused 10% (our dead-data rate is ~0.1–0.4 GiB/day
|
||||||
|
# vs ~1 T repos, so pruning can skip repacking for months) and
|
||||||
|
# --max-repack-size 4G (bounds any single Sunday's rewrite to
|
||||||
|
# ~10-15 min even after a mass deletion; the rest defers)
|
||||||
|
# - check: structure every week, plus a rotating 1/52 data subset —
|
||||||
|
# a full verification of every byte once a year
|
||||||
|
#
|
||||||
|
# Usage: restic-maintenance.sh [-r <repo>[,<repo>...]]
|
||||||
|
|
||||||
|
set -u
|
||||||
|
|
||||||
|
REPOS="s3:s3.sbg.io.cloud.ovh.net/restic-data,s3:s3.sbg.io.cloud.ovh.net/restic-incus"
|
||||||
|
LOCKFILE=/run/lock/restic-maintenance.lock
|
||||||
|
ENVFILE=/root/.restic-env
|
||||||
|
|
||||||
|
while getopts r: flag; do
|
||||||
|
case "${flag}" in
|
||||||
|
r) REPOS=${OPTARG};;
|
||||||
|
*) echo "Usage: $0 [-r <repo>[,<repo>...]]" >&2; exit 2;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
[ -r "$ENVFILE" ] || { echo "env file $ENVFILE not readable" >&2; exit 2; }
|
||||||
|
. "$ENVFILE"
|
||||||
|
|
||||||
|
exec 9>"$LOCKFILE"
|
||||||
|
if ! flock -n 9; then
|
||||||
|
echo "another restic-maintenance run holds $LOCKFILE, aborting" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
log() { echo "[$(date '+%F %T')] $*"; }
|
||||||
|
|
||||||
|
# rotate the read-data subset weekly: full coverage once a year
|
||||||
|
WEEK=$(( ($(date +%s) / 604800) % 52 + 1 ))
|
||||||
|
|
||||||
|
rc=0
|
||||||
|
for repo in $(printf '%s' "$REPOS" | tr ',' ' '); do
|
||||||
|
log "maintenance: $repo"
|
||||||
|
restic -r "$repo" unlock || rc=1
|
||||||
|
restic -r "$repo" prune --max-unused 10% --max-repack-size 4G --pack-size 64 || rc=1
|
||||||
|
restic -r "$repo" check || rc=1
|
||||||
|
log "check --read-data-subset=$WEEK/52"
|
||||||
|
restic -r "$repo" check --read-data-subset="$WEEK/52" || rc=1
|
||||||
|
done
|
||||||
|
|
||||||
|
log "done (rc=$rc)"
|
||||||
|
exit $rc
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
/var/lib/incus/storage-pools/data/containers/nextcloud/rootfs/nextcloud/data/appdata_*/preview
|
||||||
|
/var/lib/incus/storage-pools/data/containers/nextcloud/rootfs/nextcloud/data/appdata_*/dav-photocache
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
# /root/scripts/restic-paths — one path per line (--files-from-verbatim)
|
||||||
|
/var/lib/incus/storage-pools/data/containers/bitwarden/rootfs/opt/bitwarden
|
||||||
|
/var/lib/incus/storage-pools/data/containers/gateway/rootfs/var/www
|
||||||
|
/var/lib/incus/storage-pools/data/containers/git/rootfs/home/git/projects
|
||||||
|
/var/lib/incus/storage-pools/data/containers/login/rootfs/opt/authentik
|
||||||
|
/var/lib/incus/storage-pools/data/containers/mail/rootfs/var/vmail
|
||||||
|
/var/lib/incus/storage-pools/data/containers/mail/rootfs/var/www
|
||||||
|
/var/lib/incus/storage-pools/data/containers/outline/rootfs/var/lib/docker/volumes/outline_storage-data/_data
|
||||||
|
/var/lib/incus/storage-pools/data/containers/solar/rootfs/var/www/html/solar
|
||||||
|
/var/lib/incus/storage-pools/data/containers/nextcloud/rootfs/nextcloud
|
||||||
|
/var/lib/incus/storage-pools/data/containers/seafile/rootfs/opt/seafile
|
||||||
Reference in New Issue
Block a user