Rework incus-copy and incus-backup for the new two-leg backup scheme

incus-copy.sh: add -r <src-remote> (pull all of a remote's instances to
the local host) and -p <dest-project> (copy local instances into another
local project/pool, for the on-host backup leg on a second disk). Copy
all instances instead of only running ones, abort cleanly if the
instance list fails, and keep boot.autostart=false on every replica.

incus-backup.sh: add a local destination mode (-d local, now the
default) that writes dumps/rsyncs straight to the backup root without
ssh; add -b to override the /backup root, flock against overlapping
runs, set -u, non-zero exit when any step fails, and fix the invalid
`incus exec -n` flag that made MariaDB dumps fail on modern incus.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

incus-copy.sh

@@ -1,23 +1,85 @@
 #!/bin/bash
 #
+# Incremental replication of ALL Incus instances (incus copy --refresh;
+# with ZFS on both ends only deltas travel, so stopped instances cost
+# nothing after their first copy).
 # https://linuxcontainers.org/incus/docs/main/howto/move_instances/
 #
+# Three shapes, per where it runs and where the replicas land:
+#   incus-copy.sh -d <dest-remote> [-m pull|push|relay] [-s <dest-pool>]
+#       push all local instances to a remote (runs on the source host)
+#   incus-copy.sh -r <src-remote> [-m pull|push|relay] [-s <dest-pool>]
+#       pull all of a remote's instances to the local host (runs on the
+#       backup host; the source only needs its API reachable)
+#   incus-copy.sh -p <target-project> -s <dest-pool>
+#       copy all local instances into another local project + pool
+#       (on-host backup onto a different disk)
+#
+# Snapshot history/retention is handled on the source instances
+# (snapshots.schedule / snapshots.expiry); --refresh-exclude-older keeps
+# refreshes from dragging expired snapshots around.
 
-#set -x
+set -u
 
-while getopts d:m: flag
-do
+INCUS=/usr/bin/incus
+LOCKFILE=/run/lock/incus-copy.lock
+SRC=""
+DEST=""
+MODE=pull
+POOL=""
+PROJECT=""
+
+usage() {
+    echo "Usage: $0 [-d <dest-remote> | -r <src-remote>] [-m pull|push|relay] [-s <dest-storage-pool>] [-p <dest-project>]" >&2
+    exit 2
+}
+
+while getopts d:r:m:s:p: flag; do
     case "${flag}" in
         d) DEST=${OPTARG};;
+        r) SRC=${OPTARG};;
         m) MODE=${OPTARG};;
+        s) POOL=${OPTARG};;
+        p) PROJECT=${OPTARG};;
+        *) usage;;
     esac
 done
+# need somewhere for replicas to go, and one side must stay local
+[ -n "$DEST" ] || [ -n "$SRC" ] || [ -n "$PROJECT" ] || usage
+[ -n "$DEST" ] && [ -n "$SRC" ] && usage
+case "$MODE" in pull|push|relay) ;; *) usage;; esac
 
+# refuse to overlap with a previous, still-running invocation
+exec 9> "$LOCKFILE"
+if ! flock -n 9; then
+    echo "another incus-copy run holds $LOCKFILE, aborting" >&2
+    exit 1
+fi
 
-for CT in $(/usr/bin/incus list -c n -f compact,noheader status=RUNNING) ; do
-    DATE=$(date '+%Y-%m-%d %H:%M:%S')
-    CMD="/usr/bin/incus copy $CT ${DEST}:$CT --refresh --refresh-exclude-older --mode ${MODE}"
-    echo "[${DATE}] $CMD"
-    $CMD 2>&1 > /dev/null
+INSTANCES=$($INCUS list ${SRC:+"${SRC}:"} -c n -f csv) || {
+    echo "cannot list instances${SRC:+ on ${SRC}:}, aborting" >&2
+    exit 1
+}
+
+RC=0
+for CT in $INSTANCES; do
+    echo "[$(date '+%F %T')] copy ${SRC:+$SRC:}$CT -> ${DEST:+$DEST:}$CT${PROJECT:+ (project $PROJECT)}"
+    # --mode only applies to remote transfers
+    if $INCUS copy "${SRC:+$SRC:}$CT" "${DEST:+$DEST:}$CT" \
+        --refresh --refresh-exclude-older \
+        ${SRC:+--mode "$MODE"} ${DEST:+--mode "$MODE"} \
+        ${POOL:+--storage "$POOL"} \
+        ${PROJECT:+--target-project "$PROJECT"}; then
+        # the replica inherits boot.autostart from the source and would
+        # start itself on the backup host after a reboot (or, for the
+        # local-project shape, fight the live instance for its static
+        # IP on the same bridge) — neutralize it
+        $INCUS config set ${PROJECT:+--project "$PROJECT"} \
+            "${DEST:+$DEST:}$CT" boot.autostart=false || RC=1
+    else
+        echo "[$(date '+%F %T')] FAILED: $CT" >&2
+        RC=1
+    fi
 done
 
+exit $RC