diff --git a/README.md b/README.md index 710f3bc..f416705 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Some helpful bash scripts for [incus](https://linuxcontainers.org/incus) - incus-container-upgrade: Perform `apt dist-upgrade` command on running instances. -- incus-backup: Backup instances FS and DBs to a remote location using rsync over ssh. -- incus-copy: Run a differential copy of all running instances to a remote incus server. +- incus-backup: Backup instances FS and DBs to a local directory (`-d local`) or a remote location using rsync over ssh. +- incus-copy: Run a differential copy of all instances to a remote incus server (push or pull) or to a local project/pool. - incus-snapshot: Take a snapshot on all running instances. - zfs-auto-snapshot: Create and cleanup expired zfs snapshots. diff --git a/incus-backup.sh b/incus-backup.sh index 5e3fc43..005446e 100755 --- a/incus-backup.sh +++ b/incus-backup.sh @@ -1,55 +1,106 @@ #!/bin/bash # -# Backup instances FS and DBs to a remote location using rsync over ssh. +# Backup instances FS and DBs, driven by a JSON manifest (see +# incus-backup.db: {"": {"DB": [...], "FS": [...]}, ...}). +# Destination is either a remote (rsync over ssh) or, with -d local, +# a local directory (e.g. a dedicated backup zpool mounted on /backup). # +# Usage: incus-backup.sh -f -s +# [-d ] [-u ] [-i ] +# [-p ] [-b ] +# +# Layout under (default /backup): // +# plus /incus-{local,global}-db.sql -while getopts d:u:i:p:f:s: flag -do +set -u + +DEST=local +SSH_USER="" +SSH_KEY="" +SSH_PORT=22 +DB_FILE="" +STORAGE_POOL="" +BKP_ROOT=/backup +LOCKFILE=/run/lock/incus-backup.lock + +usage() { + echo "Usage: $0 -f -s [-d ] [-u ] [-i ] [-p ] [-b ]" >&2 + exit 2 +} + +while getopts d:u:i:p:f:s:b: flag; do case "${flag}" in d) DEST=${OPTARG};; - u) USER=${OPTARG};; + u) SSH_USER=${OPTARG};; i) SSH_KEY=${OPTARG};; p) SSH_PORT=${OPTARG};; f) DB_FILE=${OPTARG};; s) STORAGE_POOL=${OPTARG};; + b) BKP_ROOT=${OPTARG};; + *) usage;; esac done +[ -n "$DB_FILE" ] && [ -n "$STORAGE_POOL" ] || usage +if [ "$DEST" != "local" ]; then + [ -n "$SSH_USER" ] && [ -n "$SSH_KEY" ] || usage +fi HOST=$(hostname -s) -BKP_DIR="/backup/${HOST}" -SSH_CMD="ssh -i ${SSH_KEY} -p ${SSH_PORT}" +BKP_DIR="${BKP_ROOT}/${HOST}" CT_PREFIX="/var/lib/incus/storage-pools/${STORAGE_POOL}/containers" +# save — write stdin to a file at the destination +# transfer — rsync a path to the destination +if [ "$DEST" = "local" ]; then + save() { mkdir -p "$(dirname "$1")" && cat - > "$1"; } + transfer() { mkdir -p "$3" && /usr/bin/rsync $1 "$2" "$3"; } +else + SSH_CMD="ssh -i ${SSH_KEY} -p ${SSH_PORT}" + save() { ${SSH_CMD} "${SSH_USER}@${DEST}" "mkdir -p $(dirname "$1") && cat - > $1"; } + transfer() { /usr/bin/rsync $1 -e "${SSH_CMD}" "$2" "${SSH_USER}@${DEST}:$3"; } +fi + +# refuse to overlap with a previous, still-running invocation +exec 9> "$LOCKFILE" +if ! flock -n 9; then + echo "another incus-backup run holds $LOCKFILE, aborting" >&2 + exit 1 +fi + +RC=0 # Backup incus DB -/usr/bin/incus admin sql local .dump | ${SSH_CMD} ${USER}@${DEST} "cat - > ${BKP_DIR}/incus-local-db.sql" -/usr/bin/incus admin sql global .dump | ${SSH_CMD} ${USER}@${DEST} "cat - > ${BKP_DIR}/incus-global-db.sql" +/usr/bin/incus admin sql local .dump | save "${BKP_DIR}/incus-local-db.sql" || RC=1 +/usr/bin/incus admin sql global .dump | save "${BKP_DIR}/incus-global-db.sql" || RC=1 -for CT in $(cat ${DB_FILE} | jq -r 'keys[]') ; do +for CT in $(jq -r 'keys[]' "${DB_FILE}") ; do SRC_DIR="${CT_PREFIX}/${CT}" DST_DIR="${BKP_DIR}/${CT}" # Backup container info if [ -f "${SRC_DIR}/backup.yaml" ] ; then - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting $CT backup.yaml" - /usr/bin/rsync -a --del -e "${SSH_CMD}" ${SRC_DIR}/backup.yaml ${USER}@${DEST}:${DST_DIR}/ + echo "[$(date '+%F %T')] Starting $CT backup.yaml" + transfer "-a --del" "${SRC_DIR}/backup.yaml" "${DST_DIR}/" || RC=1 fi # Backup Mysql dumps - for DB in $(cat $DB_FILE | jq -r ".${CT} | select(.DB != null) | .DB[]") ; do - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting $CT $DB database backup" - /usr/bin/incus exec -n $CT -- mariadb-dump --single-transaction --databases ${DB} | ${SSH_CMD} ${USER}@${DEST} "mkdir -p ${DST_DIR} ; cat - > ${DST_DIR}/mysql-${DB}.sql" + for DB in $(jq -r ".${CT} | select(.DB != null) | .DB[]" "${DB_FILE}") ; do + echo "[$(date '+%F %T')] Starting $CT $DB database backup" + /usr/bin/incus exec "$CT" -- mariadb-dump --single-transaction --databases "${DB}" \ + | save "${DST_DIR}/mysql-${DB}.sql" || RC=1 done - # Backup container rootfs paths - for FS in $(cat $DB_FILE | jq -r ".${CT} | select(.FS != null) | .FS[]") ; do + # Backup container rootfs paths + for FS in $(jq -r ".${CT} | select(.FS != null) | .FS[]" "${DB_FILE}") ; do # Skip missing rootfs dir - if [ ! -d ${SRC_DIR}/rootfs ] ; then - echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARNING : Missing rootfs for container ${CT}, skipping FS ${FS} ..." + if [ ! -d "${SRC_DIR}/rootfs" ] ; then + echo "[$(date '+%F %T')] WARNING : Missing rootfs for container ${CT}, skipping FS ${FS} ..." continue fi - # "/." used by rsync to limit the amount of path information that is sent as implied directories - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting $CT $FS filesystem backup" - /usr/bin/rsync -aR --del -e "${SSH_CMD}" ${SRC_DIR}/rootfs/.${FS} ${USER}@${DEST}:${DST_DIR}/ + # "/." used by rsync to limit the amount of path information that is sent as implied directories + echo "[$(date '+%F %T')] Starting $CT $FS filesystem backup" + transfer "-aR --del" "${SRC_DIR}/rootfs/.${FS}" "${DST_DIR}/" || RC=1 done done + +exit $RC diff --git a/incus-copy.sh b/incus-copy.sh index 1e4e927..d88769a 100755 --- a/incus-copy.sh +++ b/incus-copy.sh @@ -1,23 +1,85 @@ #!/bin/bash # +# Incremental replication of ALL Incus instances (incus copy --refresh; +# with ZFS on both ends only deltas travel, so stopped instances cost +# nothing after their first copy). # https://linuxcontainers.org/incus/docs/main/howto/move_instances/ # +# Three shapes, per where it runs and where the replicas land: +# incus-copy.sh -d [-m pull|push|relay] [-s ] +# push all local instances to a remote (runs on the source host) +# incus-copy.sh -r [-m pull|push|relay] [-s ] +# pull all of a remote's instances to the local host (runs on the +# backup host; the source only needs its API reachable) +# incus-copy.sh -p -s +# copy all local instances into another local project + pool +# (on-host backup onto a different disk) +# +# Snapshot history/retention is handled on the source instances +# (snapshots.schedule / snapshots.expiry); --refresh-exclude-older keeps +# refreshes from dragging expired snapshots around. -#set -x +set -u -while getopts d:m: flag -do +INCUS=/usr/bin/incus +LOCKFILE=/run/lock/incus-copy.lock +SRC="" +DEST="" +MODE=pull +POOL="" +PROJECT="" + +usage() { + echo "Usage: $0 [-d | -r ] [-m pull|push|relay] [-s ] [-p ]" >&2 + exit 2 +} + +while getopts d:r:m:s:p: flag; do case "${flag}" in d) DEST=${OPTARG};; + r) SRC=${OPTARG};; m) MODE=${OPTARG};; + s) POOL=${OPTARG};; + p) PROJECT=${OPTARG};; + *) usage;; esac done +# need somewhere for replicas to go, and one side must stay local +[ -n "$DEST" ] || [ -n "$SRC" ] || [ -n "$PROJECT" ] || usage +[ -n "$DEST" ] && [ -n "$SRC" ] && usage +case "$MODE" in pull|push|relay) ;; *) usage;; esac +# refuse to overlap with a previous, still-running invocation +exec 9> "$LOCKFILE" +if ! flock -n 9; then + echo "another incus-copy run holds $LOCKFILE, aborting" >&2 + exit 1 +fi -for CT in $(/usr/bin/incus list -c n -f compact,noheader status=RUNNING) ; do - DATE=$(date '+%Y-%m-%d %H:%M:%S') - CMD="/usr/bin/incus copy $CT ${DEST}:$CT --refresh --refresh-exclude-older --mode ${MODE}" - echo "[${DATE}] $CMD" - $CMD 2>&1 > /dev/null +INSTANCES=$($INCUS list ${SRC:+"${SRC}:"} -c n -f csv) || { + echo "cannot list instances${SRC:+ on ${SRC}:}, aborting" >&2 + exit 1 +} + +RC=0 +for CT in $INSTANCES; do + echo "[$(date '+%F %T')] copy ${SRC:+$SRC:}$CT -> ${DEST:+$DEST:}$CT${PROJECT:+ (project $PROJECT)}" + # --mode only applies to remote transfers + if $INCUS copy "${SRC:+$SRC:}$CT" "${DEST:+$DEST:}$CT" \ + --refresh --refresh-exclude-older \ + ${SRC:+--mode "$MODE"} ${DEST:+--mode "$MODE"} \ + ${POOL:+--storage "$POOL"} \ + ${PROJECT:+--target-project "$PROJECT"}; then + # the replica inherits boot.autostart from the source and would + # start itself on the backup host after a reboot (or, for the + # local-project shape, fight the live instance for its static + # IP on the same bridge) — neutralize it + $INCUS config set ${PROJECT:+--project "$PROJECT"} \ + "${DEST:+$DEST:}$CT" boot.autostart=false || RC=1 + else + echo "[$(date '+%F %T')] FAILED: $CT" >&2 + RC=1 + fi done +exit $RC