Add restic-restore-test.sh: repeatable restore drill
Restores a filesystem tree (diffed against live) and a database dump (well-formedness, and with -d loaded into a throwaway container) from the S3 repo, then cleans up. Turns the 'restore test' release gate into something runnable each quarter instead of a one-off. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Julien Lutran
co-authored by
Claude Fable 5
Claude Fable 5
parent
bab0ae5386
commit
54ff1431b1
Executable
+101
@@ -0,0 +1,101 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Restore drill for the S3 backup leg — proves the backups are not just
|
||||
# "running" but actually restorable. Non-destructive: reads the repo,
|
||||
# writes only under a temporary directory, removes it at the end.
|
||||
#
|
||||
# Two restores, the two data kinds we back up:
|
||||
# 1. a filesystem tree -> restored, then diffed against the live tree
|
||||
# 2. a database dump -> restored, then sanity-checked (and with
|
||||
# -d, loaded into a throwaway incus container)
|
||||
#
|
||||
# Usage: restic-restore-test.sh [-r <repo>] [-t <tree-path>] [-b <db-dump>]
|
||||
# [-w <workdir>] [-k] [-d]
|
||||
# -k keep the restored files (default: clean up)
|
||||
# -d also load the dump into a scratch container (slower, strongest)
|
||||
#
|
||||
# Env from /root/.restic-env.
|
||||
|
||||
set -u
|
||||
|
||||
REPO=s3:s3.sbg.io.cloud.ovh.net/restic-data
|
||||
TREE=/var/lib/incus/storage-pools/data/containers/solar/rootfs/var/www/html/solar
|
||||
DUMP=/backup/dumps/mariadb/freshrss/freshrss.sql
|
||||
WORK=/backup/restore-test-$(date +%Y%m%d-%H%M%S)
|
||||
ENVFILE=/root/.restic-env
|
||||
RESTIC=/usr/local/bin/restic
|
||||
KEEP=0
|
||||
DBTEST=0
|
||||
|
||||
while getopts r:t:b:w:kd flag; do
|
||||
case "${flag}" in
|
||||
r) REPO=${OPTARG};; t) TREE=${OPTARG};; b) DUMP=${OPTARG};;
|
||||
w) WORK=${OPTARG};; k) KEEP=1;; d) DBTEST=1;;
|
||||
*) echo "Usage: $0 [-r repo] [-t tree] [-b dump] [-w workdir] [-k] [-d]" >&2; exit 2;;
|
||||
esac
|
||||
done
|
||||
|
||||
[ -r "$ENVFILE" ] || { echo "env file $ENVFILE not readable (run as root)" >&2; exit 2; }
|
||||
. "$ENVFILE"
|
||||
log() { echo "[$(date '+%F %T')] $*"; }
|
||||
rc=0; verdict() { [ "$1" -eq 0 ] && echo "PASS $2" || { echo "FAIL $2"; rc=1; }; }
|
||||
|
||||
mkdir -p "$WORK" || exit 2
|
||||
log "repo $REPO"
|
||||
$RESTIC -r "$REPO" snapshots --latest 1 --compact || { echo "cannot list snapshots" >&2; exit 1; }
|
||||
|
||||
# ---- 1. filesystem tree ----------------------------------------------------
|
||||
log "restoring tree: $TREE"
|
||||
$RESTIC -r "$REPO" restore latest --target "$WORK/tree" --include "$TREE" >/dev/null 2>&1
|
||||
verdict $? "tree restored"
|
||||
if [ -d "$WORK/tree$TREE" ]; then
|
||||
files=$(find "$WORK/tree$TREE" -type f | wc -l)
|
||||
log "restored $files files; diffing against the live tree"
|
||||
# differences are expected if the tree changed since the snapshot —
|
||||
# report them, do not fail on them
|
||||
if diff -qr "$WORK/tree$TREE" "$TREE" > "$WORK/diff.txt" 2>&1; then
|
||||
verdict 0 "restored tree is identical to live ($files files)"
|
||||
else
|
||||
echo "NOTE $(wc -l < "$WORK/diff.txt") path(s) differ from live (expected if changed since the snapshot):"
|
||||
head -5 "$WORK/diff.txt" | sed 's/^/ /'
|
||||
verdict 0 "restored tree readable ($files files)"
|
||||
fi
|
||||
else
|
||||
verdict 1 "restored tree missing at $WORK/tree$TREE"
|
||||
fi
|
||||
|
||||
# ---- 2. database dump ------------------------------------------------------
|
||||
log "restoring dump: $DUMP"
|
||||
$RESTIC -r "$REPO" restore latest --target "$WORK/db" --include "$DUMP" >/dev/null 2>&1
|
||||
f="$WORK/db$DUMP"
|
||||
if [ -s "$f" ]; then
|
||||
tables=$(grep -c "^CREATE TABLE" "$f")
|
||||
tail -3 "$f" | grep -q "Dump completed"
|
||||
complete=$?
|
||||
log "dump: $(du -h "$f" | cut -f1), $tables CREATE TABLE, completed-marker=$([ $complete -eq 0 ] && echo yes || echo NO)"
|
||||
verdict $(( complete != 0 || tables == 0 ? 1 : 0 )) "dump restored and well-formed"
|
||||
|
||||
if [ "$DBTEST" -eq 1 ]; then
|
||||
ct=restore-drill-$$
|
||||
log "loading it into a scratch container ($ct) — this takes a few minutes"
|
||||
if incus launch images:debian/13 "$ct" >/dev/null 2>&1 &&
|
||||
incus exec "$ct" -- sh -c "DEBIAN_FRONTEND=noninteractive apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq install -y mariadb-server" >/dev/null 2>&1; then
|
||||
incus file push "$f" "$ct/root/dump.sql" >/dev/null 2>&1
|
||||
incus exec "$ct" -- sh -c "mariadb < /root/dump.sql" && loaded=0 || loaded=1
|
||||
got=$(incus exec "$ct" -- mariadb -N -B -e \
|
||||
"SELECT COUNT(*) FROM information_schema.tables WHERE table_schema NOT IN ('mysql','information_schema','performance_schema','sys')" 2>/dev/null)
|
||||
log "tables in the restored database: ${got:-0} (dump declared $tables)"
|
||||
verdict $(( loaded != 0 || ${got:-0} == 0 ? 1 : 0 )) "dump loads into a live MariaDB"
|
||||
else
|
||||
verdict 1 "could not prepare the scratch container"
|
||||
fi
|
||||
incus delete -f "$ct" >/dev/null 2>&1
|
||||
fi
|
||||
else
|
||||
verdict 1 "dump restored at $f"
|
||||
fi
|
||||
|
||||
# ---- cleanup ---------------------------------------------------------------
|
||||
if [ "$KEEP" -eq 1 ]; then log "keeping $WORK"; else rm -rf "$WORK"; log "cleaned up $WORK"; fi
|
||||
log "restore drill done (rc=$rc)"
|
||||
exit $rc
|
||||
Reference in New Issue
Block a user