Julien LutranandClaude Fable 5
b955999553
A SUSPENDED pool keeps its vdev ONLINE, so statechange-notify never fires; data-class events need ZED_NOTIFY_DATA. Three nuc incidents went unnoticed for days. This mails on healthy<->problem transitions only, so it is silent in normal operation. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
67 lines
1.9 KiB
Bash
Executable File
67 lines
1.9 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# Pool health watchdog. ZED does not cover everything: a pool that gets
|
|
# SUSPENDED after an I/O failure keeps its vdev marked ONLINE, so
|
|
# statechange-notify.sh never fires, and data-class events are silent
|
|
# unless ZED_NOTIFY_DATA is set. Three separate incidents on nuc went
|
|
# unnoticed for days because of that.
|
|
#
|
|
# This checks `zpool status -x` and mails only on TRANSITIONS
|
|
# (healthy -> problem, problem -> healthy), so it is quiet by default
|
|
# and cannot spam. Run it often (every 15 min) from cron.
|
|
#
|
|
# Usage: zpool-health.sh [-m <mail-to>] [-s <state-file>] [-t]
|
|
# -t send a test mail and exit (proves the path works)
|
|
|
|
set -u
|
|
|
|
MAILTO=root
|
|
STATE=/var/lib/zpool-health.state
|
|
TEST=0
|
|
HOST=$(hostname -s)
|
|
|
|
while getopts m:s:t flag; do
|
|
case "${flag}" in
|
|
m) MAILTO=${OPTARG};;
|
|
s) STATE=${OPTARG};;
|
|
t) TEST=1;;
|
|
*) echo "Usage: $0 [-m <mail-to>] [-s <state-file>] [-t]" >&2; exit 2;;
|
|
esac
|
|
done
|
|
|
|
notify() { # subject, body
|
|
if command -v mail >/dev/null 2>&1; then
|
|
printf '%s\n' "$2" | mail -s "$1" "$MAILTO"
|
|
else
|
|
logger -t zpool-health "$1"
|
|
printf '%s\n' "$2" | logger -t zpool-health
|
|
fi
|
|
}
|
|
|
|
if [ "$TEST" -eq 1 ]; then
|
|
notify "[$HOST] zpool-health test" "$(zpool status -x 2>&1)"
|
|
echo "test notification sent to $MAILTO"; exit 0
|
|
fi
|
|
|
|
status=$(zpool status -x 2>&1)
|
|
if [ "$status" = "all pools are healthy" ]; then
|
|
now=ok
|
|
else
|
|
now=problem
|
|
fi
|
|
|
|
was=$(cat "$STATE" 2>/dev/null || echo ok)
|
|
printf '%s' "$now" > "$STATE"
|
|
|
|
[ "$now" = "$was" ] && exit 0 # no transition: stay quiet
|
|
|
|
if [ "$now" = problem ]; then
|
|
notify "[$HOST] ZFS POOL PROBLEM" "$(printf '%s\n\n%s\n' "$status" "$(zpool status -v 2>&1)")"
|
|
echo "$(date '+%F %T') problem: $status" >&2
|
|
exit 1
|
|
else
|
|
notify "[$HOST] ZFS pools healthy again" "$(zpool status 2>&1)"
|
|
echo "$(date '+%F %T') recovered"
|
|
fi
|
|
exit 0
|