scripts/plakar-backup.sh

#!/bin/bash
#
# Nightly plakar backup driver. Three phases:
#   1. dumps — incus's own DBs, plus application-consistent database
#      dumps taken with `incus exec` (container-local auth: no DB
#      users, no network exposure). Auto-discovered in every RUNNING
#      container — no manifest to go stale: native MariaDB
#      (mariadb-dump, all non-system DBs), native PostgreSQL (pg_dump
#      per DB + pg_dumpall --globals-only, peer auth as postgres),
#      and PostgreSQL inside docker containers (image name matching
#      "postgres", user from $POSTGRES_USER). Dumps are staged plain
#      (not gzipped, so plakar's chunking dedups near-identical
#      consecutive dumps) in the dump dir, then backed up as one
#      snapshot.
#   2. sources — every fs source listed in the sources file (one
#      `plakar source add` name per line, #-comments allowed).
#   3. retention — prune + maintenance on the kloset.
# The kloset target is whatever `plakar at` accepts — normally the S3
# store (`-k @s3`, direct-to-S3, no local staging); a path
# (e.g. -k /backup/plakar-test) only for testing. -S optionally
# replicates the kloset to a second store afterwards (`plakar sync`);
# unused in the current single-store setup.
#
# Usage: plakar-backup.sh -k <kloset> [-f <sources-file>]
#          [-r <retention-days>] [-S <sync-store>] [-d <dump-dir>]
#
# The kloset passphrase comes from $PLAKAR_PASSPHRASE or, failing
# that, /root/.plakar-passphrase (mode 600).

set -u

KLOSET=""
SRC_FILE=/root/scripts/plakar-sources
# optional DB exclude list, one dump path per line as printed by the
# "dump" log lines (<ct>/<db> or <ct>/<docker-name>/<db>); every skip
# is logged loudly so the list can't rot silently like a manifest
EXCLUDE_FILE=/root/scripts/plakar-db-exclude
RETENTION=30
SYNC_STORE=""
DUMP_DIR=/backup/plakar-dumps
LOCKFILE=/run/lock/plakar-backup.lock
PASSFILE=/root/.plakar-passphrase

usage() {
    echo "Usage: $0 -k <kloset> [-f <sources-file>] [-r <retention-days>] [-S <sync-store>] [-d <dump-dir>]" >&2
    exit 2
}

while getopts k:f:r:S:d: flag; do
    case "${flag}" in
        k) KLOSET=${OPTARG};;
        f) SRC_FILE=${OPTARG};;
        r) RETENTION=${OPTARG};;
        S) SYNC_STORE=${OPTARG};;
        d) DUMP_DIR=${OPTARG};;
        *) usage;;
    esac
done

[ -n "$KLOSET" ] || usage
[ -r "$SRC_FILE" ] || { echo "sources file $SRC_FILE not readable" >&2; exit 2; }

exec 9>"$LOCKFILE"
if ! flock -n 9; then
    echo "another plakar-backup run holds $LOCKFILE, aborting" >&2
    exit 1
fi

log() { echo "[$(date '+%F %T')] $*"; }

excluded() {
    [ -r "$EXCLUDE_FILE" ] && grep -qx "$1" "$EXCLUDE_FILE" \
        && log "SKIP $1 (listed in $EXCLUDE_FILE)"
}

if [ -z "${PLAKAR_PASSPHRASE:-}" ]; then
    [ -r "$PASSFILE" ] || { echo "no PLAKAR_PASSPHRASE in env and $PASSFILE not readable" >&2; exit 1; }
    PLAKAR_PASSPHRASE=$(cat "$PASSFILE")
fi
export PLAKAR_PASSPHRASE

rc=0

# incus's own state (instance configs, profiles, devices) — needed to
# rebuild instances, not covered by any file/DB source
mkdir -p "$DUMP_DIR/incus"
incus admin sql global .dump > "$DUMP_DIR/incus/incus-global-db.sql" || rc=1
incus admin sql local .dump > "$DUMP_DIR/incus/incus-local-db.sql" || rc=1

# Database dumps, auto-discovered per running container: native
# MariaDB, native PostgreSQL, and PostgreSQL inside docker containers
# (matched on the docker image name containing "postgres").
PG_LIST="SELECT datname FROM pg_database WHERE NOT datistemplate AND datname <> 'postgres'"
for ct in $(incus list status=running -c n -f csv); do
    # native MariaDB/MySQL (unix-socket root auth) — binaries may carry
    # either naming (e.g. mail: hand-installed mariadb-dump + mysql client)
    mdump=$(incus exec "$ct" -- sh -c 'command -v mariadb-dump || command -v mysqldump' 2>/dev/null)
    mclient=$(incus exec "$ct" -- sh -c 'command -v mariadb || command -v mysql' 2>/dev/null)
    if [ -n "$mdump" ] && [ -n "$mclient" ]; then
        dbs=$(incus exec "$ct" -- "$mclient" -N -B -e 'SHOW DATABASES') \
            || { echo "listing mariadb databases on $ct failed" >&2; rc=1; dbs=""; }
        for db in $(printf '%s\n' "$dbs" \
                    | grep -Ev '^(information_schema|performance_schema|mysql|sys)$'); do
            excluded "$ct/$db" && continue
            log "dump $ct/$db (mariadb)"
            mkdir -p "$DUMP_DIR/mariadb/$ct"
            incus exec "$ct" -- "$mdump" --single-transaction --events --routines --triggers \
                --databases "$db" > "$DUMP_DIR/mariadb/$ct/$db.sql" \
                || { echo "dump $ct/$db failed" >&2; rc=1; }
        done
        # users + grants (the mariadb equivalent of pg_dumpall
        # --globals-only): replayable SHOW GRANTS statements, portable
        # across versions unlike a raw mysql-schema dump
        log "dump $ct/grants (mariadb)"
        incus exec "$ct" -- sh -c "$mclient -NBe \"SELECT CONCAT('SHOW GRANTS FOR ', QUOTE(user), '@', QUOTE(host), ';') FROM mysql.user\" | $mclient -NB | sed 's/\$/;/'" \
            > "$DUMP_DIR/mariadb/$ct/grants.sql" \
            || { echo "grants dump on $ct failed" >&2; rc=1; }
    elif [ -n "$mdump$mclient" ]; then
        echo "$ct has only one of dump/client mariadb binaries, skipping" >&2; rc=1
    fi

    # native PostgreSQL (peer auth as the postgres user)
    if incus exec "$ct" -- sh -c 'command -v pg_dump' >/dev/null 2>&1; then
        mkdir -p "$DUMP_DIR/postgres/$ct"
        incus exec "$ct" -- su -s /bin/sh postgres -c "pg_dumpall --globals-only" \
            > "$DUMP_DIR/postgres/$ct/globals.sql" || rc=1
        dbs=$(incus exec "$ct" -- su -s /bin/sh postgres -c "psql -AtX -c \"$PG_LIST\"") \
            || { echo "listing postgres databases on $ct failed" >&2; rc=1; dbs=""; }
        for db in $dbs; do
            excluded "$ct/$db" && continue
            log "dump $ct/$db (postgres)"
            incus exec "$ct" -- su -s /bin/sh postgres -c "pg_dump --clean --if-exists $db" \
                > "$DUMP_DIR/postgres/$ct/$db.sql" \
                || { echo "dump $ct/$db failed" >&2; rc=1; }
        done
    fi

    # PostgreSQL inside docker (e.g. outline, login)
    incus exec "$ct" -- sh -c 'command -v docker' >/dev/null 2>&1 || continue
    for dc in $(incus exec "$ct" -- docker ps --format '{{.Names}} {{.Image}}' 2>/dev/null \
                | awk 'tolower($2) ~ /postgres/ {print $1}'); do
        pguser=$(incus exec "$ct" -- docker exec "$dc" sh -c 'echo "${POSTGRES_USER:-postgres}"') \
            || { echo "reading POSTGRES_USER on $ct/$dc failed" >&2; rc=1; continue; }
        mkdir -p "$DUMP_DIR/postgres/$ct/$dc"
        incus exec "$ct" -- docker exec "$dc" pg_dumpall -U "$pguser" --globals-only \
            > "$DUMP_DIR/postgres/$ct/$dc/globals.sql" || rc=1
        dbs=$(incus exec "$ct" -- docker exec "$dc" psql -U "$pguser" -AtX -c "$PG_LIST") \
            || { echo "listing postgres databases on $ct/$dc failed" >&2; rc=1; dbs=""; }
        for db in $dbs; do
            excluded "$ct/$dc/$db" && continue
            log "dump $ct/$dc/$db (postgres)"
            incus exec "$ct" -- docker exec "$dc" pg_dump -U "$pguser" --clean --if-exists "$db" \
                > "$DUMP_DIR/postgres/$ct/$dc/$db.sql" \
                || { echo "dump $ct/$dc/$db failed" >&2; rc=1; }
        done
    done
done

log "backup dumps ($DUMP_DIR)"
plakar -quiet at "$KLOSET" backup -tag dumps "$DUMP_DIR" \
    || { echo "backup of $DUMP_DIR failed" >&2; rc=1; }

for src in $(grep -Ev '^[[:space:]]*(#|$)' "$SRC_FILE"); do
    log "backup @$src"
    plakar -quiet at "$KLOSET" backup -tag "$src" "@$src" \
        || { echo "backup @$src failed" >&2; rc=1; }
done

log "prune: keep the last $RETENTION days of snapshots"
plakar at "$KLOSET" prune -days "$RETENTION" -apply || rc=1
plakar at "$KLOSET" maintenance || rc=1

if [ -n "$SYNC_STORE" ]; then
    log "sync to @$SYNC_STORE"
    plakar at "$KLOSET" sync to "@$SYNC_STORE" \
        || { echo "sync to @$SYNC_STORE failed" >&2; rc=1; }
fi

log "done (rc=$rc)"
exit $rc