Rework incus-copy and incus-backup for the new two-leg backup scheme
incus-copy.sh: add -r <src-remote> (pull all of a remote's instances to the local host) and -p <dest-project> (copy local instances into another local project/pool, for the on-host backup leg on a second disk). Copy all instances instead of only running ones, abort cleanly if the instance list fails, and keep boot.autostart=false on every replica. incus-backup.sh: add a local destination mode (-d local, now the default) that writes dumps/rsyncs straight to the backup root without ssh; add -b to override the /backup root, flock against overlapping runs, set -u, non-zero exit when any step fails, and fix the invalid `incus exec -n` flag that made MariaDB dumps fail on modern incus. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Julien Lutran
co-authored by
Claude Fable 5
Claude Fable 5
parent
f5f8dd802d
commit
856ea28d93
@@ -3,7 +3,7 @@
|
||||
Some helpful bash scripts for [incus](https://linuxcontainers.org/incus)
|
||||
|
||||
- incus-container-upgrade: Perform `apt dist-upgrade` command on running instances.
|
||||
- incus-backup: Backup instances FS and DBs to a remote location using rsync over ssh.
|
||||
- incus-copy: Run a differential copy of all running instances to a remote incus server.
|
||||
- incus-backup: Backup instances FS and DBs to a local directory (`-d local`) or a remote location using rsync over ssh.
|
||||
- incus-copy: Run a differential copy of all instances to a remote incus server (push or pull) or to a local project/pool.
|
||||
- incus-snapshot: Take a snapshot on all running instances.
|
||||
- zfs-auto-snapshot: Create and cleanup expired zfs snapshots.
|
||||
|
||||
+70
-19
@@ -1,55 +1,106 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Backup instances FS and DBs to a remote location using rsync over ssh.
|
||||
# Backup instances FS and DBs, driven by a JSON manifest (see
|
||||
# incus-backup.db: {"<instance>": {"DB": [...], "FS": [...]}, ...}).
|
||||
# Destination is either a remote (rsync over ssh) or, with -d local,
|
||||
# a local directory (e.g. a dedicated backup zpool mounted on /backup).
|
||||
#
|
||||
# Usage: incus-backup.sh -f <manifest> -s <storage-pool>
|
||||
# [-d <dest-host|local>] [-u <ssh-user>] [-i <ssh-key>]
|
||||
# [-p <ssh-port>] [-b <backup-root>]
|
||||
#
|
||||
# Layout under <backup-root> (default /backup): <hostname>/<instance>/
|
||||
# plus <hostname>/incus-{local,global}-db.sql
|
||||
|
||||
while getopts d:u:i:p:f:s: flag
|
||||
do
|
||||
set -u
|
||||
|
||||
DEST=local
|
||||
SSH_USER=""
|
||||
SSH_KEY=""
|
||||
SSH_PORT=22
|
||||
DB_FILE=""
|
||||
STORAGE_POOL=""
|
||||
BKP_ROOT=/backup
|
||||
LOCKFILE=/run/lock/incus-backup.lock
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 -f <manifest.db> -s <storage-pool> [-d <dest-host|local>] [-u <ssh-user>] [-i <ssh-key>] [-p <ssh-port>] [-b <backup-root>]" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
while getopts d:u:i:p:f:s:b: flag; do
|
||||
case "${flag}" in
|
||||
d) DEST=${OPTARG};;
|
||||
u) USER=${OPTARG};;
|
||||
u) SSH_USER=${OPTARG};;
|
||||
i) SSH_KEY=${OPTARG};;
|
||||
p) SSH_PORT=${OPTARG};;
|
||||
f) DB_FILE=${OPTARG};;
|
||||
s) STORAGE_POOL=${OPTARG};;
|
||||
b) BKP_ROOT=${OPTARG};;
|
||||
*) usage;;
|
||||
esac
|
||||
done
|
||||
[ -n "$DB_FILE" ] && [ -n "$STORAGE_POOL" ] || usage
|
||||
if [ "$DEST" != "local" ]; then
|
||||
[ -n "$SSH_USER" ] && [ -n "$SSH_KEY" ] || usage
|
||||
fi
|
||||
|
||||
HOST=$(hostname -s)
|
||||
BKP_DIR="/backup/${HOST}"
|
||||
SSH_CMD="ssh -i ${SSH_KEY} -p ${SSH_PORT}"
|
||||
BKP_DIR="${BKP_ROOT}/${HOST}"
|
||||
CT_PREFIX="/var/lib/incus/storage-pools/${STORAGE_POOL}/containers"
|
||||
|
||||
# save <dst-file> — write stdin to a file at the destination
|
||||
# transfer <opts> <src> <dst-dir> — rsync a path to the destination
|
||||
if [ "$DEST" = "local" ]; then
|
||||
save() { mkdir -p "$(dirname "$1")" && cat - > "$1"; }
|
||||
transfer() { mkdir -p "$3" && /usr/bin/rsync $1 "$2" "$3"; }
|
||||
else
|
||||
SSH_CMD="ssh -i ${SSH_KEY} -p ${SSH_PORT}"
|
||||
save() { ${SSH_CMD} "${SSH_USER}@${DEST}" "mkdir -p $(dirname "$1") && cat - > $1"; }
|
||||
transfer() { /usr/bin/rsync $1 -e "${SSH_CMD}" "$2" "${SSH_USER}@${DEST}:$3"; }
|
||||
fi
|
||||
|
||||
# refuse to overlap with a previous, still-running invocation
|
||||
exec 9> "$LOCKFILE"
|
||||
if ! flock -n 9; then
|
||||
echo "another incus-backup run holds $LOCKFILE, aborting" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
RC=0
|
||||
|
||||
# Backup incus DB
|
||||
/usr/bin/incus admin sql local .dump | ${SSH_CMD} ${USER}@${DEST} "cat - > ${BKP_DIR}/incus-local-db.sql"
|
||||
/usr/bin/incus admin sql global .dump | ${SSH_CMD} ${USER}@${DEST} "cat - > ${BKP_DIR}/incus-global-db.sql"
|
||||
/usr/bin/incus admin sql local .dump | save "${BKP_DIR}/incus-local-db.sql" || RC=1
|
||||
/usr/bin/incus admin sql global .dump | save "${BKP_DIR}/incus-global-db.sql" || RC=1
|
||||
|
||||
for CT in $(cat ${DB_FILE} | jq -r 'keys[]') ; do
|
||||
for CT in $(jq -r 'keys[]' "${DB_FILE}") ; do
|
||||
SRC_DIR="${CT_PREFIX}/${CT}"
|
||||
DST_DIR="${BKP_DIR}/${CT}"
|
||||
|
||||
# Backup container info
|
||||
if [ -f "${SRC_DIR}/backup.yaml" ] ; then
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting $CT backup.yaml"
|
||||
/usr/bin/rsync -a --del -e "${SSH_CMD}" ${SRC_DIR}/backup.yaml ${USER}@${DEST}:${DST_DIR}/
|
||||
echo "[$(date '+%F %T')] Starting $CT backup.yaml"
|
||||
transfer "-a --del" "${SRC_DIR}/backup.yaml" "${DST_DIR}/" || RC=1
|
||||
fi
|
||||
|
||||
# Backup Mysql dumps
|
||||
for DB in $(cat $DB_FILE | jq -r ".${CT} | select(.DB != null) | .DB[]") ; do
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting $CT $DB database backup"
|
||||
/usr/bin/incus exec -n $CT -- mariadb-dump --single-transaction --databases ${DB} | ${SSH_CMD} ${USER}@${DEST} "mkdir -p ${DST_DIR} ; cat - > ${DST_DIR}/mysql-${DB}.sql"
|
||||
for DB in $(jq -r ".${CT} | select(.DB != null) | .DB[]" "${DB_FILE}") ; do
|
||||
echo "[$(date '+%F %T')] Starting $CT $DB database backup"
|
||||
/usr/bin/incus exec "$CT" -- mariadb-dump --single-transaction --databases "${DB}" \
|
||||
| save "${DST_DIR}/mysql-${DB}.sql" || RC=1
|
||||
done
|
||||
|
||||
# Backup container rootfs paths
|
||||
for FS in $(cat $DB_FILE | jq -r ".${CT} | select(.FS != null) | .FS[]") ; do
|
||||
for FS in $(jq -r ".${CT} | select(.FS != null) | .FS[]" "${DB_FILE}") ; do
|
||||
# Skip missing rootfs dir
|
||||
if [ ! -d ${SRC_DIR}/rootfs ] ; then
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARNING : Missing rootfs for container ${CT}, skipping FS ${FS} ..."
|
||||
if [ ! -d "${SRC_DIR}/rootfs" ] ; then
|
||||
echo "[$(date '+%F %T')] WARNING : Missing rootfs for container ${CT}, skipping FS ${FS} ..."
|
||||
continue
|
||||
fi
|
||||
# "/." used by rsync to limit the amount of path information that is sent as implied directories
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting $CT $FS filesystem backup"
|
||||
/usr/bin/rsync -aR --del -e "${SSH_CMD}" ${SRC_DIR}/rootfs/.${FS} ${USER}@${DEST}:${DST_DIR}/
|
||||
echo "[$(date '+%F %T')] Starting $CT $FS filesystem backup"
|
||||
transfer "-aR --del" "${SRC_DIR}/rootfs/.${FS}" "${DST_DIR}/" || RC=1
|
||||
done
|
||||
done
|
||||
|
||||
exit $RC
|
||||
|
||||
+70
-8
@@ -1,23 +1,85 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Incremental replication of ALL Incus instances (incus copy --refresh;
|
||||
# with ZFS on both ends only deltas travel, so stopped instances cost
|
||||
# nothing after their first copy).
|
||||
# https://linuxcontainers.org/incus/docs/main/howto/move_instances/
|
||||
#
|
||||
# Three shapes, per where it runs and where the replicas land:
|
||||
# incus-copy.sh -d <dest-remote> [-m pull|push|relay] [-s <dest-pool>]
|
||||
# push all local instances to a remote (runs on the source host)
|
||||
# incus-copy.sh -r <src-remote> [-m pull|push|relay] [-s <dest-pool>]
|
||||
# pull all of a remote's instances to the local host (runs on the
|
||||
# backup host; the source only needs its API reachable)
|
||||
# incus-copy.sh -p <target-project> -s <dest-pool>
|
||||
# copy all local instances into another local project + pool
|
||||
# (on-host backup onto a different disk)
|
||||
#
|
||||
# Snapshot history/retention is handled on the source instances
|
||||
# (snapshots.schedule / snapshots.expiry); --refresh-exclude-older keeps
|
||||
# refreshes from dragging expired snapshots around.
|
||||
|
||||
#set -x
|
||||
set -u
|
||||
|
||||
while getopts d:m: flag
|
||||
do
|
||||
INCUS=/usr/bin/incus
|
||||
LOCKFILE=/run/lock/incus-copy.lock
|
||||
SRC=""
|
||||
DEST=""
|
||||
MODE=pull
|
||||
POOL=""
|
||||
PROJECT=""
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 [-d <dest-remote> | -r <src-remote>] [-m pull|push|relay] [-s <dest-storage-pool>] [-p <dest-project>]" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
while getopts d:r:m:s:p: flag; do
|
||||
case "${flag}" in
|
||||
d) DEST=${OPTARG};;
|
||||
r) SRC=${OPTARG};;
|
||||
m) MODE=${OPTARG};;
|
||||
s) POOL=${OPTARG};;
|
||||
p) PROJECT=${OPTARG};;
|
||||
*) usage;;
|
||||
esac
|
||||
done
|
||||
# need somewhere for replicas to go, and one side must stay local
|
||||
[ -n "$DEST" ] || [ -n "$SRC" ] || [ -n "$PROJECT" ] || usage
|
||||
[ -n "$DEST" ] && [ -n "$SRC" ] && usage
|
||||
case "$MODE" in pull|push|relay) ;; *) usage;; esac
|
||||
|
||||
# refuse to overlap with a previous, still-running invocation
|
||||
exec 9> "$LOCKFILE"
|
||||
if ! flock -n 9; then
|
||||
echo "another incus-copy run holds $LOCKFILE, aborting" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for CT in $(/usr/bin/incus list -c n -f compact,noheader status=RUNNING) ; do
|
||||
DATE=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
CMD="/usr/bin/incus copy $CT ${DEST}:$CT --refresh --refresh-exclude-older --mode ${MODE}"
|
||||
echo "[${DATE}] $CMD"
|
||||
$CMD 2>&1 > /dev/null
|
||||
INSTANCES=$($INCUS list ${SRC:+"${SRC}:"} -c n -f csv) || {
|
||||
echo "cannot list instances${SRC:+ on ${SRC}:}, aborting" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
RC=0
|
||||
for CT in $INSTANCES; do
|
||||
echo "[$(date '+%F %T')] copy ${SRC:+$SRC:}$CT -> ${DEST:+$DEST:}$CT${PROJECT:+ (project $PROJECT)}"
|
||||
# --mode only applies to remote transfers
|
||||
if $INCUS copy "${SRC:+$SRC:}$CT" "${DEST:+$DEST:}$CT" \
|
||||
--refresh --refresh-exclude-older \
|
||||
${SRC:+--mode "$MODE"} ${DEST:+--mode "$MODE"} \
|
||||
${POOL:+--storage "$POOL"} \
|
||||
${PROJECT:+--target-project "$PROJECT"}; then
|
||||
# the replica inherits boot.autostart from the source and would
|
||||
# start itself on the backup host after a reboot (or, for the
|
||||
# local-project shape, fight the live instance for its static
|
||||
# IP on the same bridge) — neutralize it
|
||||
$INCUS config set ${PROJECT:+--project "$PROJECT"} \
|
||||
"${DEST:+$DEST:}$CT" boot.autostart=false || RC=1
|
||||
else
|
||||
echo "[$(date '+%F %T')] FAILED: $CT" >&2
|
||||
RC=1
|
||||
fi
|
||||
done
|
||||
|
||||
exit $RC
|
||||
|
||||
Reference in New Issue
Block a user