Rework incus-copy and incus-backup for the new two-leg backup scheme

incus-copy.sh: add -r <src-remote> (pull all of a remote's instances to
the local host) and -p <dest-project> (copy local instances into another
local project/pool, for the on-host backup leg on a second disk). Copy
all instances instead of only running ones, abort cleanly if the
instance list fails, and keep boot.autostart=false on every replica.

incus-backup.sh: add a local destination mode (-d local, now the
default) that writes dumps/rsyncs straight to the backup root without
ssh; add -b to override the /backup root, flock against overlapping
runs, set -u, non-zero exit when any step fails, and fix the invalid
`incus exec -n` flag that made MariaDB dumps fail on modern incus.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

README.md

@@ -3,7 +3,7 @@
 Some helpful bash scripts for [incus](https://linuxcontainers.org/incus)
 
 - incus-container-upgrade: Perform `apt dist-upgrade` command on running instances.
-- incus-backup: Backup instances FS and DBs to a remote location using rsync over ssh.
+- incus-backup: Backup instances FS and DBs to a local directory (`-d local`) or a remote location using rsync over ssh.
-- incus-copy: Run a differential copy of all running instances to a remote incus server.
+- incus-copy: Run a differential copy of all instances to a remote incus server (push or pull) or to a local project/pool.
 - incus-snapshot: Take a snapshot on all running instances.
 - zfs-auto-snapshot: Create and cleanup expired zfs snapshots.

incus-backup.sh

@@ -1,55 +1,106 @@
 #!/bin/bash
 #
-# Backup instances FS and DBs to a remote location using rsync over ssh.
+# Backup instances FS and DBs, driven by a JSON manifest (see
+# incus-backup.db: {"<instance>": {"DB": [...], "FS": [...]}, ...}).
+# Destination is either a remote (rsync over ssh) or, with -d local,
+# a local directory (e.g. a dedicated backup zpool mounted on /backup).
 #
+# Usage: incus-backup.sh -f <manifest> -s <storage-pool>
+#          [-d <dest-host|local>] [-u <ssh-user>] [-i <ssh-key>]
+#          [-p <ssh-port>] [-b <backup-root>]
+#
+# Layout under <backup-root> (default /backup): <hostname>/<instance>/
+#   plus <hostname>/incus-{local,global}-db.sql
 
-while getopts d:u:i:p:f:s: flag
+set -u
-do
+
+DEST=local
+SSH_USER=""
+SSH_KEY=""
+SSH_PORT=22
+DB_FILE=""
+STORAGE_POOL=""
+BKP_ROOT=/backup
+LOCKFILE=/run/lock/incus-backup.lock
+
+usage() {
+    echo "Usage: $0 -f <manifest.db> -s <storage-pool> [-d <dest-host|local>] [-u <ssh-user>] [-i <ssh-key>] [-p <ssh-port>] [-b <backup-root>]" >&2
+    exit 2
+}
+
+while getopts d:u:i:p:f:s:b: flag; do
     case "${flag}" in
         d) DEST=${OPTARG};;
-        u) USER=${OPTARG};;
+        u) SSH_USER=${OPTARG};;
         i) SSH_KEY=${OPTARG};;
         p) SSH_PORT=${OPTARG};;
         f) DB_FILE=${OPTARG};;
         s) STORAGE_POOL=${OPTARG};;
+        b) BKP_ROOT=${OPTARG};;
+        *) usage;;
     esac
 done
+[ -n "$DB_FILE" ] && [ -n "$STORAGE_POOL" ] || usage
+if [ "$DEST" != "local" ]; then
+    [ -n "$SSH_USER" ] && [ -n "$SSH_KEY" ] || usage
+fi
 
 HOST=$(hostname -s)
-BKP_DIR="/backup/${HOST}"
+BKP_DIR="${BKP_ROOT}/${HOST}"
-SSH_CMD="ssh -i ${SSH_KEY} -p ${SSH_PORT}"
 CT_PREFIX="/var/lib/incus/storage-pools/${STORAGE_POOL}/containers"
 
+# save <dst-file>          — write stdin to a file at the destination
+# transfer <opts> <src> <dst-dir>  — rsync a path to the destination
+if [ "$DEST" = "local" ]; then
+    save()     { mkdir -p "$(dirname "$1")" && cat - > "$1"; }
+    transfer() { mkdir -p "$3" && /usr/bin/rsync $1 "$2" "$3"; }
+else
+    SSH_CMD="ssh -i ${SSH_KEY} -p ${SSH_PORT}"
+    save()     { ${SSH_CMD} "${SSH_USER}@${DEST}" "mkdir -p $(dirname "$1") && cat - > $1"; }
+    transfer() { /usr/bin/rsync $1 -e "${SSH_CMD}" "$2" "${SSH_USER}@${DEST}:$3"; }
+fi
+
+# refuse to overlap with a previous, still-running invocation
+exec 9> "$LOCKFILE"
+if ! flock -n 9; then
+    echo "another incus-backup run holds $LOCKFILE, aborting" >&2
+    exit 1
+fi
+
+RC=0
 
 # Backup incus DB
-/usr/bin/incus admin sql local .dump | ${SSH_CMD} ${USER}@${DEST} "cat - > ${BKP_DIR}/incus-local-db.sql"
+/usr/bin/incus admin sql local .dump  | save "${BKP_DIR}/incus-local-db.sql"  || RC=1
-/usr/bin/incus admin sql global .dump | ${SSH_CMD} ${USER}@${DEST} "cat - > ${BKP_DIR}/incus-global-db.sql"
+/usr/bin/incus admin sql global .dump | save "${BKP_DIR}/incus-global-db.sql" || RC=1
 
-for CT in $(cat ${DB_FILE} | jq -r 'keys[]') ; do
+for CT in $(jq -r 'keys[]' "${DB_FILE}") ; do
     SRC_DIR="${CT_PREFIX}/${CT}"
     DST_DIR="${BKP_DIR}/${CT}"
 
     # Backup container info
     if [ -f "${SRC_DIR}/backup.yaml" ] ; then
-        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting $CT backup.yaml"
+        echo "[$(date '+%F %T')] Starting $CT backup.yaml"
-        /usr/bin/rsync -a --del -e "${SSH_CMD}" ${SRC_DIR}/backup.yaml ${USER}@${DEST}:${DST_DIR}/
+        transfer "-a --del" "${SRC_DIR}/backup.yaml" "${DST_DIR}/" || RC=1
     fi
 
     # Backup Mysql dumps
-    for DB in $(cat $DB_FILE | jq -r ".${CT} | select(.DB != null) | .DB[]") ; do
+    for DB in $(jq -r ".${CT} | select(.DB != null) | .DB[]" "${DB_FILE}") ; do
-        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting $CT $DB database backup"
+        echo "[$(date '+%F %T')] Starting $CT $DB database backup"
-        /usr/bin/incus exec -n $CT -- mariadb-dump --single-transaction --databases ${DB} | ${SSH_CMD} ${USER}@${DEST} "mkdir -p ${DST_DIR} ; cat - > ${DST_DIR}/mysql-${DB}.sql"
+        /usr/bin/incus exec "$CT" -- mariadb-dump --single-transaction --databases "${DB}" \
+            | save "${DST_DIR}/mysql-${DB}.sql" || RC=1
     done
 
-    # Backup container rootfs paths 
+    # Backup container rootfs paths
-    for FS in $(cat $DB_FILE | jq -r ".${CT} | select(.FS != null) | .FS[]") ; do
+    for FS in $(jq -r ".${CT} | select(.FS != null) | .FS[]" "${DB_FILE}") ; do
         # Skip missing rootfs dir
-        if [ ! -d ${SRC_DIR}/rootfs ] ; then
+        if [ ! -d "${SRC_DIR}/rootfs" ] ; then
-            echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARNING : Missing rootfs for container ${CT}, skipping FS ${FS} ..."
+            echo "[$(date '+%F %T')] WARNING : Missing rootfs for container ${CT}, skipping FS ${FS} ..."
             continue
         fi
-	    # "/." used by rsync to limit the amount of path information that is sent as implied directories
+        # "/." used by rsync to limit the amount of path information that is sent as implied directories
-        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting $CT $FS filesystem backup"
+        echo "[$(date '+%F %T')] Starting $CT $FS filesystem backup"
-	    /usr/bin/rsync -aR --del -e "${SSH_CMD}" ${SRC_DIR}/rootfs/.${FS} ${USER}@${DEST}:${DST_DIR}/
+        transfer "-aR --del" "${SRC_DIR}/rootfs/.${FS}" "${DST_DIR}/" || RC=1
     done
 done
+
+exit $RC

incus-copy.sh

@@ -1,23 +1,85 @@
 #!/bin/bash
 #
+# Incremental replication of ALL Incus instances (incus copy --refresh;
+# with ZFS on both ends only deltas travel, so stopped instances cost
+# nothing after their first copy).
 # https://linuxcontainers.org/incus/docs/main/howto/move_instances/
 #
+# Three shapes, per where it runs and where the replicas land:
+#   incus-copy.sh -d <dest-remote> [-m pull|push|relay] [-s <dest-pool>]
+#       push all local instances to a remote (runs on the source host)
+#   incus-copy.sh -r <src-remote> [-m pull|push|relay] [-s <dest-pool>]
+#       pull all of a remote's instances to the local host (runs on the
+#       backup host; the source only needs its API reachable)
+#   incus-copy.sh -p <target-project> -s <dest-pool>
+#       copy all local instances into another local project + pool
+#       (on-host backup onto a different disk)
+#
+# Snapshot history/retention is handled on the source instances
+# (snapshots.schedule / snapshots.expiry); --refresh-exclude-older keeps
+# refreshes from dragging expired snapshots around.
 
-#set -x
+set -u
 
-while getopts d:m: flag
+INCUS=/usr/bin/incus
-do
+LOCKFILE=/run/lock/incus-copy.lock
+SRC=""
+DEST=""
+MODE=pull
+POOL=""
+PROJECT=""
+
+usage() {
+    echo "Usage: $0 [-d <dest-remote> | -r <src-remote>] [-m pull|push|relay] [-s <dest-storage-pool>] [-p <dest-project>]" >&2
+    exit 2
+}
+
+while getopts d:r:m:s:p: flag; do
     case "${flag}" in
         d) DEST=${OPTARG};;
+        r) SRC=${OPTARG};;
         m) MODE=${OPTARG};;
+        s) POOL=${OPTARG};;
+        p) PROJECT=${OPTARG};;
+        *) usage;;
     esac
 done
+# need somewhere for replicas to go, and one side must stay local
+[ -n "$DEST" ] || [ -n "$SRC" ] || [ -n "$PROJECT" ] || usage
+[ -n "$DEST" ] && [ -n "$SRC" ] && usage
+case "$MODE" in pull|push|relay) ;; *) usage;; esac
 
+# refuse to overlap with a previous, still-running invocation
+exec 9> "$LOCKFILE"
+if ! flock -n 9; then
+    echo "another incus-copy run holds $LOCKFILE, aborting" >&2
+    exit 1
+fi
 
-for CT in $(/usr/bin/incus list -c n -f compact,noheader status=RUNNING) ; do
+INSTANCES=$($INCUS list ${SRC:+"${SRC}:"} -c n -f csv) || {
-    DATE=$(date '+%Y-%m-%d %H:%M:%S')
+    echo "cannot list instances${SRC:+ on ${SRC}:}, aborting" >&2
-    CMD="/usr/bin/incus copy $CT ${DEST}:$CT --refresh --refresh-exclude-older --mode ${MODE}"
+    exit 1
-    echo "[${DATE}] $CMD"
+}
-    $CMD 2>&1 > /dev/null
+
+RC=0
+for CT in $INSTANCES; do
+    echo "[$(date '+%F %T')] copy ${SRC:+$SRC:}$CT -> ${DEST:+$DEST:}$CT${PROJECT:+ (project $PROJECT)}"
+    # --mode only applies to remote transfers
+    if $INCUS copy "${SRC:+$SRC:}$CT" "${DEST:+$DEST:}$CT" \
+        --refresh --refresh-exclude-older \
+        ${SRC:+--mode "$MODE"} ${DEST:+--mode "$MODE"} \
+        ${POOL:+--storage "$POOL"} \
+        ${PROJECT:+--target-project "$PROJECT"}; then
+        # the replica inherits boot.autostart from the source and would
+        # start itself on the backup host after a reboot (or, for the
+        # local-project shape, fight the live instance for its static
+        # IP on the same bridge) — neutralize it
+        $INCUS config set ${PROJECT:+--project "$PROJECT"} \
+            "${DEST:+$DEST:}$CT" boot.autostart=false || RC=1
+    else
+        echo "[$(date '+%F %T')] FAILED: $CT" >&2
+        RC=1
+    fi
 done
 
+exit $RC