From 54ff1431b1676fa40f742c34128933f1b218b747 Mon Sep 17 00:00:00 2001
From: Julien Lutran <julien.lutran@ovhcloud.com>
Date: Fri, 28 Aug 2026 18:42:33 +0200
Subject: [PATCH] Add restic-restore-test.sh: repeatable restore drill

Restores a filesystem tree (diffed against live) and a database dump
(well-formedness, and with -d loaded into a throwaway container) from
the S3 repo, then cleans up. Turns the 'restore test' release gate
into something runnable each quarter instead of a one-off.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 restic-restore-test.sh | 101 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 101 insertions(+)
 create mode 100755 restic-restore-test.sh

diff --git a/restic-restore-test.sh b/restic-restore-test.sh
new file mode 100755
index 0000000..f93e054
--- /dev/null
+++ b/restic-restore-test.sh
@@ -0,0 +1,101 @@
+#!/bin/bash
+#
+# Restore drill for the S3 backup leg — proves the backups are not just
+# "running" but actually restorable. Non-destructive: reads the repo,
+# writes only under a temporary directory, removes it at the end.
+#
+# Two restores, the two data kinds we back up:
+#   1. a filesystem tree   -> restored, then diffed against the live tree
+#   2. a database dump     -> restored, then sanity-checked (and with
+#                             -d, loaded into a throwaway incus container)
+#
+# Usage: restic-restore-test.sh [-r <repo>] [-t <tree-path>] [-b <db-dump>]
+#                               [-w <workdir>] [-k] [-d]
+#   -k  keep the restored files (default: clean up)
+#   -d  also load the dump into a scratch container (slower, strongest)
+#
+# Env from /root/.restic-env.
+
+set -u
+
+REPO=s3:s3.sbg.io.cloud.ovh.net/restic-data
+TREE=/var/lib/incus/storage-pools/data/containers/solar/rootfs/var/www/html/solar
+DUMP=/backup/dumps/mariadb/freshrss/freshrss.sql
+WORK=/backup/restore-test-$(date +%Y%m%d-%H%M%S)
+ENVFILE=/root/.restic-env
+RESTIC=/usr/local/bin/restic
+KEEP=0
+DBTEST=0
+
+while getopts r:t:b:w:kd flag; do
+    case "${flag}" in
+        r) REPO=${OPTARG};; t) TREE=${OPTARG};; b) DUMP=${OPTARG};;
+        w) WORK=${OPTARG};; k) KEEP=1;; d) DBTEST=1;;
+        *) echo "Usage: $0 [-r repo] [-t tree] [-b dump] [-w workdir] [-k] [-d]" >&2; exit 2;;
+    esac
+done
+
+[ -r "$ENVFILE" ] || { echo "env file $ENVFILE not readable (run as root)" >&2; exit 2; }
+. "$ENVFILE"
+log() { echo "[$(date '+%F %T')] $*"; }
+rc=0; verdict() { [ "$1" -eq 0 ] && echo "PASS  $2" || { echo "FAIL  $2"; rc=1; }; }
+
+mkdir -p "$WORK" || exit 2
+log "repo $REPO"
+$RESTIC -r "$REPO" snapshots --latest 1 --compact || { echo "cannot list snapshots" >&2; exit 1; }
+
+# ---- 1. filesystem tree ----------------------------------------------------
+log "restoring tree: $TREE"
+$RESTIC -r "$REPO" restore latest --target "$WORK/tree" --include "$TREE" >/dev/null 2>&1
+verdict $? "tree restored"
+if [ -d "$WORK/tree$TREE" ]; then
+    files=$(find "$WORK/tree$TREE" -type f | wc -l)
+    log "restored $files files; diffing against the live tree"
+    # differences are expected if the tree changed since the snapshot —
+    # report them, do not fail on them
+    if diff -qr "$WORK/tree$TREE" "$TREE" > "$WORK/diff.txt" 2>&1; then
+        verdict 0 "restored tree is identical to live ($files files)"
+    else
+        echo "NOTE  $(wc -l < "$WORK/diff.txt") path(s) differ from live (expected if changed since the snapshot):"
+        head -5 "$WORK/diff.txt" | sed 's/^/      /'
+        verdict 0 "restored tree readable ($files files)"
+    fi
+else
+    verdict 1 "restored tree missing at $WORK/tree$TREE"
+fi
+
+# ---- 2. database dump ------------------------------------------------------
+log "restoring dump: $DUMP"
+$RESTIC -r "$REPO" restore latest --target "$WORK/db" --include "$DUMP" >/dev/null 2>&1
+f="$WORK/db$DUMP"
+if [ -s "$f" ]; then
+    tables=$(grep -c "^CREATE TABLE" "$f")
+    tail -3 "$f" | grep -q "Dump completed"
+    complete=$?
+    log "dump: $(du -h "$f" | cut -f1), $tables CREATE TABLE, completed-marker=$([ $complete -eq 0 ] && echo yes || echo NO)"
+    verdict $(( complete != 0 || tables == 0 ? 1 : 0 )) "dump restored and well-formed"
+
+    if [ "$DBTEST" -eq 1 ]; then
+        ct=restore-drill-$$
+        log "loading it into a scratch container ($ct) — this takes a few minutes"
+        if incus launch images:debian/13 "$ct" >/dev/null 2>&1 &&
+           incus exec "$ct" -- sh -c "DEBIAN_FRONTEND=noninteractive apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq install -y mariadb-server" >/dev/null 2>&1; then
+            incus file push "$f" "$ct/root/dump.sql" >/dev/null 2>&1
+            incus exec "$ct" -- sh -c "mariadb < /root/dump.sql" && loaded=0 || loaded=1
+            got=$(incus exec "$ct" -- mariadb -N -B -e \
+                 "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema NOT IN ('mysql','information_schema','performance_schema','sys')" 2>/dev/null)
+            log "tables in the restored database: ${got:-0} (dump declared $tables)"
+            verdict $(( loaded != 0 || ${got:-0} == 0 ? 1 : 0 )) "dump loads into a live MariaDB"
+        else
+            verdict 1 "could not prepare the scratch container"
+        fi
+        incus delete -f "$ct" >/dev/null 2>&1
+    fi
+else
+    verdict 1 "dump restored at $f"
+fi
+
+# ---- cleanup ---------------------------------------------------------------
+if [ "$KEEP" -eq 1 ]; then log "keeping $WORK"; else rm -rf "$WORK"; log "cleaned up $WORK"; fi
+log "restore drill done (rc=$rc)"
+exit $rc