Julien LutranandClaude Fable 5
54ff1431b1
Restores a filesystem tree (diffed against live) and a database dump (well-formedness, and with -d loaded into a throwaway container) from the S3 repo, then cleans up. Turns the 'restore test' release gate into something runnable each quarter instead of a one-off. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
102 lines
4.5 KiB
Bash
Executable File
102 lines
4.5 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# Restore drill for the S3 backup leg — proves the backups are not just
|
|
# "running" but actually restorable. Non-destructive: reads the repo,
|
|
# writes only under a temporary directory, removes it at the end.
|
|
#
|
|
# Two restores, the two data kinds we back up:
|
|
# 1. a filesystem tree -> restored, then diffed against the live tree
|
|
# 2. a database dump -> restored, then sanity-checked (and with
|
|
# -d, loaded into a throwaway incus container)
|
|
#
|
|
# Usage: restic-restore-test.sh [-r <repo>] [-t <tree-path>] [-b <db-dump>]
|
|
# [-w <workdir>] [-k] [-d]
|
|
# -k keep the restored files (default: clean up)
|
|
# -d also load the dump into a scratch container (slower, strongest)
|
|
#
|
|
# Env from /root/.restic-env.
|
|
|
|
set -u
|
|
|
|
REPO=s3:s3.sbg.io.cloud.ovh.net/restic-data
|
|
TREE=/var/lib/incus/storage-pools/data/containers/solar/rootfs/var/www/html/solar
|
|
DUMP=/backup/dumps/mariadb/freshrss/freshrss.sql
|
|
WORK=/backup/restore-test-$(date +%Y%m%d-%H%M%S)
|
|
ENVFILE=/root/.restic-env
|
|
RESTIC=/usr/local/bin/restic
|
|
KEEP=0
|
|
DBTEST=0
|
|
|
|
while getopts r:t:b:w:kd flag; do
|
|
case "${flag}" in
|
|
r) REPO=${OPTARG};; t) TREE=${OPTARG};; b) DUMP=${OPTARG};;
|
|
w) WORK=${OPTARG};; k) KEEP=1;; d) DBTEST=1;;
|
|
*) echo "Usage: $0 [-r repo] [-t tree] [-b dump] [-w workdir] [-k] [-d]" >&2; exit 2;;
|
|
esac
|
|
done
|
|
|
|
[ -r "$ENVFILE" ] || { echo "env file $ENVFILE not readable (run as root)" >&2; exit 2; }
|
|
. "$ENVFILE"
|
|
log() { echo "[$(date '+%F %T')] $*"; }
|
|
rc=0; verdict() { [ "$1" -eq 0 ] && echo "PASS $2" || { echo "FAIL $2"; rc=1; }; }
|
|
|
|
mkdir -p "$WORK" || exit 2
|
|
log "repo $REPO"
|
|
$RESTIC -r "$REPO" snapshots --latest 1 --compact || { echo "cannot list snapshots" >&2; exit 1; }
|
|
|
|
# ---- 1. filesystem tree ----------------------------------------------------
|
|
log "restoring tree: $TREE"
|
|
$RESTIC -r "$REPO" restore latest --target "$WORK/tree" --include "$TREE" >/dev/null 2>&1
|
|
verdict $? "tree restored"
|
|
if [ -d "$WORK/tree$TREE" ]; then
|
|
files=$(find "$WORK/tree$TREE" -type f | wc -l)
|
|
log "restored $files files; diffing against the live tree"
|
|
# differences are expected if the tree changed since the snapshot —
|
|
# report them, do not fail on them
|
|
if diff -qr "$WORK/tree$TREE" "$TREE" > "$WORK/diff.txt" 2>&1; then
|
|
verdict 0 "restored tree is identical to live ($files files)"
|
|
else
|
|
echo "NOTE $(wc -l < "$WORK/diff.txt") path(s) differ from live (expected if changed since the snapshot):"
|
|
head -5 "$WORK/diff.txt" | sed 's/^/ /'
|
|
verdict 0 "restored tree readable ($files files)"
|
|
fi
|
|
else
|
|
verdict 1 "restored tree missing at $WORK/tree$TREE"
|
|
fi
|
|
|
|
# ---- 2. database dump ------------------------------------------------------
|
|
log "restoring dump: $DUMP"
|
|
$RESTIC -r "$REPO" restore latest --target "$WORK/db" --include "$DUMP" >/dev/null 2>&1
|
|
f="$WORK/db$DUMP"
|
|
if [ -s "$f" ]; then
|
|
tables=$(grep -c "^CREATE TABLE" "$f")
|
|
tail -3 "$f" | grep -q "Dump completed"
|
|
complete=$?
|
|
log "dump: $(du -h "$f" | cut -f1), $tables CREATE TABLE, completed-marker=$([ $complete -eq 0 ] && echo yes || echo NO)"
|
|
verdict $(( complete != 0 || tables == 0 ? 1 : 0 )) "dump restored and well-formed"
|
|
|
|
if [ "$DBTEST" -eq 1 ]; then
|
|
ct=restore-drill-$$
|
|
log "loading it into a scratch container ($ct) — this takes a few minutes"
|
|
if incus launch images:debian/13 "$ct" >/dev/null 2>&1 &&
|
|
incus exec "$ct" -- sh -c "DEBIAN_FRONTEND=noninteractive apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq install -y mariadb-server" >/dev/null 2>&1; then
|
|
incus file push "$f" "$ct/root/dump.sql" >/dev/null 2>&1
|
|
incus exec "$ct" -- sh -c "mariadb < /root/dump.sql" && loaded=0 || loaded=1
|
|
got=$(incus exec "$ct" -- mariadb -N -B -e \
|
|
"SELECT COUNT(*) FROM information_schema.tables WHERE table_schema NOT IN ('mysql','information_schema','performance_schema','sys')" 2>/dev/null)
|
|
log "tables in the restored database: ${got:-0} (dump declared $tables)"
|
|
verdict $(( loaded != 0 || ${got:-0} == 0 ? 1 : 0 )) "dump loads into a live MariaDB"
|
|
else
|
|
verdict 1 "could not prepare the scratch container"
|
|
fi
|
|
incus delete -f "$ct" >/dev/null 2>&1
|
|
fi
|
|
else
|
|
verdict 1 "dump restored at $f"
|
|
fi
|
|
|
|
# ---- cleanup ---------------------------------------------------------------
|
|
if [ "$KEEP" -eq 1 ]; then log "keeping $WORK"; else rm -rf "$WORK"; log "cleaned up $WORK"; fi
|
|
log "restore drill done (rc=$rc)"
|
|
exit $rc
|