#!/bin/bash
#
# Nightly restic backup of the incus instances (repo: restic-incus).
# For every instance of the `backup` project (quiesced replicas —
# stopped, refreshed by the 01:00 incus-copy), mount its filesystem
# with `incus file mount` (FUSE over the per-instance sftp API, works
# on stopped containers, needs sshfs) and back it up per-file,
# together with its expanded config.
#
# ⚠️ Each instance gets its OWN mountpoint (/run/restic-incus/<name>):
# restic selects a snapshot's parent by host+path, so a shared
# mountpoint would parent every snapshot on the previous *other*
# instance and force nightly full re-reads.
#
# Runs CHAINED after incus-copy.sh in the same cron entry — the
# snapshot is only as fresh as the last completed replica refresh:
#   0 1 * * * incus-copy.sh -p backup -s backup >> /var/log/incus-copy.log 2>&1 ; restic-incus-backup.sh >> /var/log/restic-incus.log 2>&1
#
# Usage: restic-incus-backup.sh [-r <repo>] [-p <project>]
#          [-x <exclude,list>] [-i <only,these>]
#
# All instances by default; opt-out via -x (logged loudly — the list
# cannot rot silently).

set -u

REPO=s3:s3.sbg.io.cloud.ovh.net/restic-incus
PROJECT=backup
EXCLUDE_INSTANCES="nextcloud,seafile"   # seek-bound giants: data covered by restic-data
ONLY_INSTANCES=""
MNT_ROOT=/run/restic-incus
LOCKFILE=/run/lock/restic-incus-backup.lock
ENVFILE=/root/.restic-env
RESTIC=/usr/local/bin/restic   # cron PATH lacks /usr/local/bin
MOUNT_TIMEOUT=30

usage() {
    echo "Usage: $0 [-r <repo>] [-p <project>] [-x <exclude,list>] [-i <only,list>]" >&2
    exit 2
}

while getopts r:p:x:i: flag; do
    case "${flag}" in
        r) REPO=${OPTARG};;
        p) PROJECT=${OPTARG};;
        x) EXCLUDE_INSTANCES=${OPTARG};;
        i) ONLY_INSTANCES=${OPTARG};;
        *) usage;;
    esac
done

[ -r "$ENVFILE" ] || { echo "env file $ENVFILE not readable" >&2; exit 2; }
. "$ENVFILE"
command -v sshfs >/dev/null || { echo "sshfs not installed (needed by incus file mount)" >&2; exit 2; }

exec 9>"$LOCKFILE"
if ! flock -n 9; then
    echo "another restic-incus-backup run holds $LOCKFILE, aborting" >&2
    exit 1
fi

log() { echo "[$(date '+%F %T')] $*"; }

cleanup_mount() {  # $1 = mountpoint, $2 = mount pid
    [ -n "${2:-}" ] && kill "$2" 2>/dev/null
    for _ in 1 2 3 4 5; do
        mountpoint -q "$1" || return 0
        fusermount -u "$1" 2>/dev/null || umount "$1" 2>/dev/null
        sleep 1
    done
    mountpoint -q "$1" && { echo "failed to unmount $1" >&2; return 1; }
    return 0
}

rc=0
mkdir -p "$MNT_ROOT"

for inst in $(incus list --project "$PROJECT" -c n -f csv); do
    if [ -n "$ONLY_INSTANCES" ]; then
        case ",$ONLY_INSTANCES," in *",$inst,"*) ;; *) continue;; esac
    fi
    case ",$EXCLUDE_INSTANCES," in
        *",$inst,"*) log "SKIP $inst (excluded)"; continue;;
    esac

    mnt="$MNT_ROOT/$inst"
    mkdir -p "$mnt"
    mountpoint -q "$mnt" && cleanup_mount "$mnt" ""   # stale from a killed run

    # instance definition, backed up alongside the tree
    incus config show "$inst" --project "$PROJECT" --expanded > "$MNT_ROOT/$inst.yaml" \
        || { echo "config dump of $inst failed" >&2; rc=1; }

    incus file mount "$inst/" "$mnt" --project "$PROJECT" >/dev/null 2>&1 &
    mpid=$!
    mounted=""
    for _ in $(seq "$MOUNT_TIMEOUT"); do
        mountpoint -q "$mnt" && { mounted=1; break; }
        kill -0 "$mpid" 2>/dev/null || break
        sleep 1
    done
    if [ -z "$mounted" ]; then
        echo "mount of $inst failed" >&2; rc=1
        cleanup_mount "$mnt" "$mpid"
        continue
    fi

    log "backup $inst"
    $RESTIC -r "$REPO" backup \
        --pack-size 64 --read-concurrency 8 -o s3.connections=8 \
        --tag "$inst" "$mnt" "$MNT_ROOT/$inst.yaml" \
        || { echo "backup of $inst failed" >&2; rc=1; }

    cleanup_mount "$mnt" "$mpid" || rc=1
done

log "forget: keep 14d/8w/6m"
$RESTIC -r "$REPO" forget --keep-daily 14 --keep-weekly 8 --keep-monthly 6 || rc=1

log "done (rc=$rc)"
exit $rc