#!/bin/bash
#
# apt dist-upgrade of all RUNNING containers (containers only — VMs are
# excluded, they may not run an agent or apt at all). Containers without
# apt are skipped. Exits non-zero if any upgrade failed.
#
# Also refreshes `user.os` / `user.os-checked` on each container, so
#   incus list -c n,config:user.os,config:user.os-checked
# always shows the OS the container actually runs. (`image.description`
# is deliberately left alone: it records what the instance was created
# from — 2019 images for most of this fleet — which is history worth
# keeping, not an inventory.)
#
# Usage: incus-container-upgrade.sh [-o]     # -o: only refresh user.os,
#                                            #     skip the upgrades

set -uo pipefail

INCUS=/usr/bin/incus
LOG=/var/log/incus-container-upgrade.log
LOCKFILE=/run/lock/incus-container-upgrade.lock
OSONLY=0

while getopts o flag; do
    case "${flag}" in
        o) OSONLY=1;;
        *) echo "Usage: $0 [-o]" >&2; exit 2;;
    esac
done

# refuse to overlap with a previous, still-running invocation
exec 9> "$LOCKFILE"
if ! flock -n 9; then
    echo "another incus-container-upgrade run holds $LOCKFILE, aborting" >&2
    exit 1
fi

distUpgrade() {
    local CT=$1
    echo -e "\n*** [$(date '+%F %T')] - Dist-upgrading $CT container ***\n"
    if ! $INCUS exec "$CT" -- sh -c 'command -v apt-get >/dev/null'; then
        echo "$CT: no apt-get in container, skipping"
        return 0
    fi
    $INCUS exec "$CT" --env DEBIAN_FRONTEND=noninteractive -- apt-get -qq update &&
    $INCUS exec "$CT" -- sh -c 'apt list --upgradable 2>/dev/null' &&
    $INCUS exec "$CT" --env DEBIAN_FRONTEND=noninteractive -- apt-get -qq -y dist-upgrade
}

# record the OS the container actually runs; failures here never fail
# the upgrade (it is metadata, not the job)
recordOS() {
    local CT=$1 OS
    OS=$($INCUS exec "$CT" -- sh -c '. /etc/os-release 2>/dev/null && echo "$PRETTY_NAME"' 2>/dev/null)
    [ -n "$OS" ] || return 0
    $INCUS config set "$CT" user.os="$OS" user.os-checked="$(date +%F)" 2>/dev/null \
        || echo "$CT: could not record user.os" >&2
}

RC=0
for CT in $($INCUS list -c n -f csv status=RUNNING type=container); do
    if [ "$OSONLY" -eq 0 ]; then
        if ! distUpgrade "$CT" 2>&1 | tee -a "$LOG"; then
            echo "[$(date '+%F %T')] FAILED: $CT" | tee -a "$LOG" >&2
            RC=1
        fi
    fi
    recordOS "$CT"
done

exit $RC