julien/scripts
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

zpool-health: remind while unhealthy, so a stuck alarm cannot mask new problems

Browse Source 
Transition-only notification has a blind spot spotted on nas: while
stale <metadata> entries keep 'zpool status -x' unhealthy, the
watchdog parks in the alarm state and can never signal anything new.
Now it re-notifies every -i hours (default 24) while the problem
persists.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Julien Lutran
2026-08-31 10:36:36 +02:00
co-authored by Claude Fable 5
parent b955999553
commit 0cd1e03b7b
1 changed files with 30 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
zpool-health.sh
+30 -10
View File
@@ -6,26 +6,32 @@
# unless ZED_NOTIFY_DATA is set. Three separate incidents on nuc went # unless ZED_NOTIFY_DATA is set. Three separate incidents on nuc went
# unnoticed for days because of that. # unnoticed for days because of that.
# #
# This checks `zpool status -x` and mails only on TRANSITIONS # This checks `zpool status -x` and mails on TRANSITIONS
# (healthy -> problem, problem -> healthy), so it is quiet by default # (healthy -> problem, problem -> healthy), plus a low-rate reminder
# and cannot spam. Run it often (every 15 min) from cron. # while a problem persists — otherwise a pool stuck unhealthy (e.g.
# stale `<metadata>` entries in the error log) parks the watchdog in
# the alarm state, where it can no longer signal anything NEW.
# Quiet by default, cannot spam. Run every 15 min from cron.
# #
# Usage: zpool-health.sh [-m <mail-to>] [-s <state-file>] [-t] # Usage: zpool-health.sh [-m <mail-to>] [-s <state-file>] [-i <hours>] [-t]
# -i hours between reminders while unhealthy (default 24, 0=off)
# -t send a test mail and exit (proves the path works) # -t send a test mail and exit (proves the path works)
set -u set -u
MAILTO=root MAILTO=root
STATE=/var/lib/zpool-health.state STATE=/var/lib/zpool-health.state
REMIND_H=24
TEST=0 TEST=0
HOST=$(hostname -s) HOST=$(hostname -s)
while getopts m:s:t flag; do while getopts m:s:i:t flag; do
case "${flag}" in case "${flag}" in
m) MAILTO=${OPTARG};; m) MAILTO=${OPTARG};;
s) STATE=${OPTARG};; s) STATE=${OPTARG};;
i) REMIND_H=${OPTARG};;
t) TEST=1;; t) TEST=1;;
*) echo "Usage: $0 [-m <mail-to>] [-s <state-file>] [-t]" >&2; exit 2;; *) echo "Usage: $0 [-m <mail-to>] [-s <state-file>] [-i <hours>] [-t]" >&2; exit 2;;
esac esac
done done
@@ -50,13 +56,27 @@ else
now=problem now=problem
fi fi
was=$(cat "$STATE" 2>/dev/null || echo ok) # state file: "<status> <epoch of last notification>"
printf '%s' "$now" > "$STATE" read -r was last < "$STATE" 2>/dev/null || { was=ok; last=0; }
[ -n "${last:-}" ] || last=0
nowsec=$(date +%s)
[ "$now" = "$was" ] && exit 0 # no transition: stay quiet remind=0
if [ "$now" = problem ] && [ "$was" = problem ] && [ "$REMIND_H" -gt 0 ]; then
[ $(( nowsec - last )) -ge $(( REMIND_H * 3600 )) ] && remind=1
fi
if [ "$now" = "$was" ] && [ "$remind" -eq 0 ]; then
printf '%s %s' "$now" "$last" > "$STATE" # keep the notify time
[ "$now" = problem ] && exit 1 || exit 0
fi
printf '%s %s' "$now" "$nowsec" > "$STATE"
if [ "$now" = problem ]; then if [ "$now" = problem ]; then
notify "[$HOST] ZFS POOL PROBLEM" "$(printf '%s\n\n%s\n' "$status" "$(zpool status -v 2>&1)")" subj="[$HOST] ZFS POOL PROBLEM"
[ "$remind" -eq 1 ] && subj="[$HOST] ZFS pool STILL unhealthy (${REMIND_H}h reminder)"
notify "$subj" "$(printf '%s\n\n%s\n' "$status" "$(zpool status -v 2>&1)")"
echo "$(date '+%F %T') problem: $status" >&2 echo "$(date '+%F %T') problem: $status" >&2
exit 1 exit 1
else else