#!/bin/bash # # Restore drill for the S3 backup leg — proves the backups are not just # "running" but actually restorable. Non-destructive: reads the repo, # writes only under a temporary directory, removes it at the end. # # Two restores, the two data kinds we back up: # 1. a filesystem tree -> restored, then diffed against the live tree # 2. a database dump -> restored, then sanity-checked (and with # -d, loaded into a throwaway incus container) # # Usage: restic-restore-test.sh [-r ] [-t ] [-b ] # [-w ] [-k] [-d] # -k keep the restored files (default: clean up) # -d also load the dump into a scratch container (slower, strongest) # # Env from /root/.restic-env. set -u REPO=s3:s3.sbg.io.cloud.ovh.net/restic-data TREE=/var/lib/incus/storage-pools/data/containers/solar/rootfs/var/www/html/solar DUMP=/backup/dumps/mariadb/freshrss/freshrss.sql WORK=/backup/restore-test-$(date +%Y%m%d-%H%M%S) ENVFILE=/root/.restic-env RESTIC=/usr/local/bin/restic KEEP=0 DBTEST=0 while getopts r:t:b:w:kd flag; do case "${flag}" in r) REPO=${OPTARG};; t) TREE=${OPTARG};; b) DUMP=${OPTARG};; w) WORK=${OPTARG};; k) KEEP=1;; d) DBTEST=1;; *) echo "Usage: $0 [-r repo] [-t tree] [-b dump] [-w workdir] [-k] [-d]" >&2; exit 2;; esac done [ -r "$ENVFILE" ] || { echo "env file $ENVFILE not readable (run as root)" >&2; exit 2; } . "$ENVFILE" log() { echo "[$(date '+%F %T')] $*"; } rc=0; verdict() { [ "$1" -eq 0 ] && echo "PASS $2" || { echo "FAIL $2"; rc=1; }; } mkdir -p "$WORK" || exit 2 log "repo $REPO" $RESTIC -r "$REPO" snapshots --latest 1 --compact || { echo "cannot list snapshots" >&2; exit 1; } # ---- 1. filesystem tree ---------------------------------------------------- log "restoring tree: $TREE" $RESTIC -r "$REPO" restore latest --target "$WORK/tree" --include "$TREE" >/dev/null 2>&1 verdict $? "tree restored" if [ -d "$WORK/tree$TREE" ]; then files=$(find "$WORK/tree$TREE" -type f | wc -l) log "restored $files files; diffing against the live tree" # differences are expected if the tree changed since the snapshot — # report them, do not fail on them if diff -qr "$WORK/tree$TREE" "$TREE" > "$WORK/diff.txt" 2>&1; then verdict 0 "restored tree is identical to live ($files files)" else echo "NOTE $(wc -l < "$WORK/diff.txt") path(s) differ from live (expected if changed since the snapshot):" head -5 "$WORK/diff.txt" | sed 's/^/ /' verdict 0 "restored tree readable ($files files)" fi else verdict 1 "restored tree missing at $WORK/tree$TREE" fi # ---- 2. database dump ------------------------------------------------------ log "restoring dump: $DUMP" $RESTIC -r "$REPO" restore latest --target "$WORK/db" --include "$DUMP" >/dev/null 2>&1 f="$WORK/db$DUMP" if [ -s "$f" ]; then tables=$(grep -c "^CREATE TABLE" "$f") tail -3 "$f" | grep -q "Dump completed" complete=$? log "dump: $(du -h "$f" | cut -f1), $tables CREATE TABLE, completed-marker=$([ $complete -eq 0 ] && echo yes || echo NO)" verdict $(( complete != 0 || tables == 0 ? 1 : 0 )) "dump restored and well-formed" if [ "$DBTEST" -eq 1 ]; then ct=restore-drill-$$ log "loading it into a scratch container ($ct) — this takes a few minutes" if incus launch images:debian/13 "$ct" >/dev/null 2>&1 && incus exec "$ct" -- sh -c "DEBIAN_FRONTEND=noninteractive apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq install -y mariadb-server" >/dev/null 2>&1; then incus file push "$f" "$ct/root/dump.sql" >/dev/null 2>&1 incus exec "$ct" -- sh -c "mariadb < /root/dump.sql" && loaded=0 || loaded=1 got=$(incus exec "$ct" -- mariadb -N -B -e \ "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema NOT IN ('mysql','information_schema','performance_schema','sys')" 2>/dev/null) log "tables in the restored database: ${got:-0} (dump declared $tables)" verdict $(( loaded != 0 || ${got:-0} == 0 ? 1 : 0 )) "dump loads into a live MariaDB" else verdict 1 "could not prepare the scratch container" fi incus delete -f "$ct" >/dev/null 2>&1 fi else verdict 1 "dump restored at $f" fi # ---- cleanup --------------------------------------------------------------- if [ "$KEEP" -eq 1 ]; then log "keeping $WORK"; else rm -rf "$WORK"; log "cleaned up $WORK"; fi log "restore drill done (rc=$rc)" exit $rc