#!/bin/bash
#
# Pool health watchdog. ZED does not cover everything: a pool that gets
# SUSPENDED after an I/O failure keeps its vdev marked ONLINE, so
# statechange-notify.sh never fires, and data-class events are silent
# unless ZED_NOTIFY_DATA is set. Three separate incidents on nuc went
# unnoticed for days because of that.
#
# This checks `zpool status -x` and mails on TRANSITIONS
# (healthy -> problem, problem -> healthy), plus a low-rate reminder
# while a problem persists — otherwise a pool stuck unhealthy (e.g.
# stale `<metadata>` entries in the error log) parks the watchdog in
# the alarm state, where it can no longer signal anything NEW.
# Quiet by default, cannot spam. Run every 15 min from cron.
#
# Usage: zpool-health.sh [-m <mail-to>] [-s <state-file>] [-i <hours>] [-t]
#          -i  hours between reminders while unhealthy (default 24, 0=off)
#          -t  send a test mail and exit (proves the path works)

set -u

MAILTO=root
STATE=/var/lib/zpool-health.state
REMIND_H=24
TEST=0
HOST=$(hostname -s)

while getopts m:s:i:t flag; do
    case "${flag}" in
        m) MAILTO=${OPTARG};;
        s) STATE=${OPTARG};;
        i) REMIND_H=${OPTARG};;
        t) TEST=1;;
        *) echo "Usage: $0 [-m <mail-to>] [-s <state-file>] [-i <hours>] [-t]" >&2; exit 2;;
    esac
done

notify() {  # subject, body
    if command -v mail >/dev/null 2>&1; then
        printf '%s\n' "$2" | mail -s "$1" "$MAILTO"
    else
        logger -t zpool-health "$1"
        printf '%s\n' "$2" | logger -t zpool-health
    fi
}

if [ "$TEST" -eq 1 ]; then
    notify "[$HOST] zpool-health test" "$(zpool status -x 2>&1)"
    echo "test notification sent to $MAILTO"; exit 0
fi

status=$(zpool status -x 2>&1)
if [ "$status" = "all pools are healthy" ]; then
    now=ok
else
    now=problem
fi

# state file: "<status> <epoch of last notification>"
read -r was last < "$STATE" 2>/dev/null || { was=ok; last=0; }
[ -n "${last:-}" ] || last=0
nowsec=$(date +%s)

remind=0
if [ "$now" = problem ] && [ "$was" = problem ] && [ "$REMIND_H" -gt 0 ]; then
    [ $(( nowsec - last )) -ge $(( REMIND_H * 3600 )) ] && remind=1
fi

if [ "$now" = "$was" ] && [ "$remind" -eq 0 ]; then
    printf '%s %s' "$now" "$last" > "$STATE"      # keep the notify time
    [ "$now" = problem ] && exit 1 || exit 0
fi

printf '%s %s' "$now" "$nowsec" > "$STATE"

if [ "$now" = problem ]; then
    subj="[$HOST] ZFS POOL PROBLEM"
    [ "$remind" -eq 1 ] && subj="[$HOST] ZFS pool STILL unhealthy (${REMIND_H}h reminder)"
    notify "$subj" "$(printf '%s\n\n%s\n' "$status" "$(zpool status -v 2>&1)")"
    echo "$(date '+%F %T') problem: $status" >&2
    exit 1
else
    notify "[$HOST] ZFS pools healthy again" "$(zpool status 2>&1)"
    echo "$(date '+%F %T') recovered"
fi
exit 0