Julien LutranandClaude Fable 5
0cd1e03b7b
Transition-only notification has a blind spot spotted on nas: while stale <metadata> entries keep 'zpool status -x' unhealthy, the watchdog parks in the alarm state and can never signal anything new. Now it re-notifies every -i hours (default 24) while the problem persists. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
87 lines
2.7 KiB
Bash
Executable File
87 lines
2.7 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# Pool health watchdog. ZED does not cover everything: a pool that gets
|
|
# SUSPENDED after an I/O failure keeps its vdev marked ONLINE, so
|
|
# statechange-notify.sh never fires, and data-class events are silent
|
|
# unless ZED_NOTIFY_DATA is set. Three separate incidents on nuc went
|
|
# unnoticed for days because of that.
|
|
#
|
|
# This checks `zpool status -x` and mails on TRANSITIONS
|
|
# (healthy -> problem, problem -> healthy), plus a low-rate reminder
|
|
# while a problem persists — otherwise a pool stuck unhealthy (e.g.
|
|
# stale `<metadata>` entries in the error log) parks the watchdog in
|
|
# the alarm state, where it can no longer signal anything NEW.
|
|
# Quiet by default, cannot spam. Run every 15 min from cron.
|
|
#
|
|
# Usage: zpool-health.sh [-m <mail-to>] [-s <state-file>] [-i <hours>] [-t]
|
|
# -i hours between reminders while unhealthy (default 24, 0=off)
|
|
# -t send a test mail and exit (proves the path works)
|
|
|
|
set -u
|
|
|
|
MAILTO=root
|
|
STATE=/var/lib/zpool-health.state
|
|
REMIND_H=24
|
|
TEST=0
|
|
HOST=$(hostname -s)
|
|
|
|
while getopts m:s:i:t flag; do
|
|
case "${flag}" in
|
|
m) MAILTO=${OPTARG};;
|
|
s) STATE=${OPTARG};;
|
|
i) REMIND_H=${OPTARG};;
|
|
t) TEST=1;;
|
|
*) echo "Usage: $0 [-m <mail-to>] [-s <state-file>] [-i <hours>] [-t]" >&2; exit 2;;
|
|
esac
|
|
done
|
|
|
|
notify() { # subject, body
|
|
if command -v mail >/dev/null 2>&1; then
|
|
printf '%s\n' "$2" | mail -s "$1" "$MAILTO"
|
|
else
|
|
logger -t zpool-health "$1"
|
|
printf '%s\n' "$2" | logger -t zpool-health
|
|
fi
|
|
}
|
|
|
|
if [ "$TEST" -eq 1 ]; then
|
|
notify "[$HOST] zpool-health test" "$(zpool status -x 2>&1)"
|
|
echo "test notification sent to $MAILTO"; exit 0
|
|
fi
|
|
|
|
status=$(zpool status -x 2>&1)
|
|
if [ "$status" = "all pools are healthy" ]; then
|
|
now=ok
|
|
else
|
|
now=problem
|
|
fi
|
|
|
|
# state file: "<status> <epoch of last notification>"
|
|
read -r was last < "$STATE" 2>/dev/null || { was=ok; last=0; }
|
|
[ -n "${last:-}" ] || last=0
|
|
nowsec=$(date +%s)
|
|
|
|
remind=0
|
|
if [ "$now" = problem ] && [ "$was" = problem ] && [ "$REMIND_H" -gt 0 ]; then
|
|
[ $(( nowsec - last )) -ge $(( REMIND_H * 3600 )) ] && remind=1
|
|
fi
|
|
|
|
if [ "$now" = "$was" ] && [ "$remind" -eq 0 ]; then
|
|
printf '%s %s' "$now" "$last" > "$STATE" # keep the notify time
|
|
[ "$now" = problem ] && exit 1 || exit 0
|
|
fi
|
|
|
|
printf '%s %s' "$now" "$nowsec" > "$STATE"
|
|
|
|
if [ "$now" = problem ]; then
|
|
subj="[$HOST] ZFS POOL PROBLEM"
|
|
[ "$remind" -eq 1 ] && subj="[$HOST] ZFS pool STILL unhealthy (${REMIND_H}h reminder)"
|
|
notify "$subj" "$(printf '%s\n\n%s\n' "$status" "$(zpool status -v 2>&1)")"
|
|
echo "$(date '+%F %T') problem: $status" >&2
|
|
exit 1
|
|
else
|
|
notify "[$HOST] ZFS pools healthy again" "$(zpool status 2>&1)"
|
|
echo "$(date '+%F %T') recovered"
|
|
fi
|
|
exit 0
|