From 0cd1e03b7b11f13c2a2e1d28a2904a9c60d3eb85 Mon Sep 17 00:00:00 2001
From: Julien Lutran <julien.lutran@ovhcloud.com>
Date: Mon, 31 Aug 2026 10:36:36 +0200
Subject: [PATCH] zpool-health: remind while unhealthy, so a stuck alarm cannot
 mask new problems

Transition-only notification has a blind spot spotted on nas: while
stale <metadata> entries keep 'zpool status -x' unhealthy, the
watchdog parks in the alarm state and can never signal anything new.
Now it re-notifies every -i hours (default 24) while the problem
persists.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 zpool-health.sh | 40 ++++++++++++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 10 deletions(-)

diff --git a/zpool-health.sh b/zpool-health.sh
index b2d2030..6bc886d 100755
--- a/zpool-health.sh
+++ b/zpool-health.sh
@@ -6,26 +6,32 @@
 # unless ZED_NOTIFY_DATA is set. Three separate incidents on nuc went
 # unnoticed for days because of that.
 #
-# This checks `zpool status -x` and mails only on TRANSITIONS
-# (healthy -> problem, problem -> healthy), so it is quiet by default
-# and cannot spam. Run it often (every 15 min) from cron.
+# This checks `zpool status -x` and mails on TRANSITIONS
+# (healthy -> problem, problem -> healthy), plus a low-rate reminder
+# while a problem persists — otherwise a pool stuck unhealthy (e.g.
+# stale `<metadata>` entries in the error log) parks the watchdog in
+# the alarm state, where it can no longer signal anything NEW.
+# Quiet by default, cannot spam. Run every 15 min from cron.
 #
-# Usage: zpool-health.sh [-m <mail-to>] [-s <state-file>] [-t]
+# Usage: zpool-health.sh [-m <mail-to>] [-s <state-file>] [-i <hours>] [-t]
+#          -i  hours between reminders while unhealthy (default 24, 0=off)
 #          -t  send a test mail and exit (proves the path works)
 
 set -u
 
 MAILTO=root
 STATE=/var/lib/zpool-health.state
+REMIND_H=24
 TEST=0
 HOST=$(hostname -s)
 
-while getopts m:s:t flag; do
+while getopts m:s:i:t flag; do
     case "${flag}" in
         m) MAILTO=${OPTARG};;
         s) STATE=${OPTARG};;
+        i) REMIND_H=${OPTARG};;
         t) TEST=1;;
-        *) echo "Usage: $0 [-m <mail-to>] [-s <state-file>] [-t]" >&2; exit 2;;
+        *) echo "Usage: $0 [-m <mail-to>] [-s <state-file>] [-i <hours>] [-t]" >&2; exit 2;;
     esac
 done
 
@@ -50,13 +56,27 @@ else
     now=problem
 fi
 
-was=$(cat "$STATE" 2>/dev/null || echo ok)
-printf '%s' "$now" > "$STATE"
+# state file: "<status> <epoch of last notification>"
+read -r was last < "$STATE" 2>/dev/null || { was=ok; last=0; }
+[ -n "${last:-}" ] || last=0
+nowsec=$(date +%s)
 
-[ "$now" = "$was" ] && exit 0   # no transition: stay quiet
+remind=0
+if [ "$now" = problem ] && [ "$was" = problem ] && [ "$REMIND_H" -gt 0 ]; then
+    [ $(( nowsec - last )) -ge $(( REMIND_H * 3600 )) ] && remind=1
+fi
+
+if [ "$now" = "$was" ] && [ "$remind" -eq 0 ]; then
+    printf '%s %s' "$now" "$last" > "$STATE"      # keep the notify time
+    [ "$now" = problem ] && exit 1 || exit 0
+fi
+
+printf '%s %s' "$now" "$nowsec" > "$STATE"
 
 if [ "$now" = problem ]; then
-    notify "[$HOST] ZFS POOL PROBLEM" "$(printf '%s\n\n%s\n' "$status" "$(zpool status -v 2>&1)")"
+    subj="[$HOST] ZFS POOL PROBLEM"
+    [ "$remind" -eq 1 ] && subj="[$HOST] ZFS pool STILL unhealthy (${REMIND_H}h reminder)"
+    notify "$subj" "$(printf '%s\n\n%s\n' "$status" "$(zpool status -v 2>&1)")"
     echo "$(date '+%F %T') problem: $status" >&2
     exit 1
 else