#!/bin/sh
#
# wan-watch.sh — one script, one log, immediate verdict every run.
#
# Deploy (Pi + monstro + FE):
#   sudo cp wan-watch.sh /usr/local/bin/wan-watch.sh
#   sudo chmod +x /usr/local/bin/wan-watch.sh
#   sudo cp wan-watch.env.example /etc/conf.d/wan-watch
#
# Cron (every 5 min — fast enough to catch 15-min pulses):
#   */5 * * * * root . /etc/conf.d/wan-watch 2>/dev/null; /usr/local/bin/wan-watch.sh >>/var/log/wan-watch.cron.log 2>&1
#
# Manual:
#   wan-watch.sh              # cron snapshot + probes + verdict
#   wan-watch.sh --full       # human-readable dump NOW (no wait)
#   wan-watch.sh --analyze    # conclusion from log (last 24h default)
#   wan-watch.sh --timeline   # hourly good/bad bars
#   wan-watch.sh --force-deep # full deep capture even if OK
#
set -eu

WAN_WATCH_LOG="${WAN_WATCH_LOG:-/var/log/wan-watch.log}"
WAN_WATCH_ROLE="${WAN_WATCH_ROLE:-${WAN_SNAPSHOT_ROLE:-auto}}"
WAN_WATCH_LOCK="${WAN_WATCH_LOCK:-/var/run/wan-watch.lock}"
WAN_WATCH_COOLDOWN_S="${WAN_WATCH_COOLDOWN_S:-300}"
WAN_WATCH_COOLDOWN_FILE="${WAN_WATCH_COOLDOWN_FILE:-/var/run/wan-watch-deep.stamp}"
WAN_WATCH_DEEP="${WAN_WATCH_DEEP:-auto}"
WAN_ANALYZE_HOURS="${WAN_ANALYZE_HOURS:-24}"

WAN_PING_TARGET="${WAN_PING_TARGET:-1.1.1.1}"
FE_PING_TARGET="${FE_PING_TARGET:-10.7.0.10}"
BE_PING_TARGET="${BE_PING_TARGET-10.7.16.128}"
ROUTER_PING_TARGET="${ROUTER_PING_TARGET:-none}"
GW_PING_TARGET="${GW_PING_TARGET:-auto}"

PING_COUNT="${PING_COUNT:-3}"
PING_DEADLINE_S="${PING_DEADLINE_S:-6}"
DEEP_PING_COUNT="${DEEP_PING_COUNT:-8}"
DEEP_PING_DEADLINE_S="${DEEP_PING_DEADLINE_S:-10}"

WAN_ALERT_MS="${WAN_ALERT_MS:-50}"
FE_ALERT_MS="${FE_ALERT_MS:-50}"
BE_ALERT_MS="${BE_ALERT_MS:-50}"
GW_ALERT_MS="${GW_ALERT_MS:-50}"
ROUTER_ALERT_MS="${ROUTER_ALERT_MS:-50}"
LOSS_ALERT_PCT="${LOSS_ALERT_PCT:-1}"
MDEV_ALERT_MS="${MDEV_ALERT_MS:-80}"
CONNTRACK_ALERT_PCT="${CONNTRACK_ALERT_PCT:-70}"

# ---------------------------------------------------------------------------
emit() {
  _kind="$1"; _ts="$2"; _role="$3"; _host="$4"
  shift 4
  _line="kind=$_kind ts=$_ts ver=3 role=$_role host=$_host $*"
  _dir="$(dirname "$WAN_WATCH_LOG")"
  mkdir -p "$_dir" 2>/dev/null || true
  printf '%s\n' "$_line" >> "$WAN_WATCH_LOG"
}

detect_role() {
  _role="$WAN_WATCH_ROLE"
  if [ "$_role" != "auto" ]; then printf '%s' "$_role"; return; fi
  _hn="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)"
  case "$_hn" in
    *raspberry*|*rpi*) printf 'pi' ;;
    *monstro*|*router*|*gw*) printf 'router' ;;
    *gentoo*|*wg0*|*fe*) printf 'fe' ;;
    *) printf 'host' ;;
  esac
}

default_gw() { ip route show default 2>/dev/null | awk '{print $3; exit}'; }

route_dev() {
  ip route get "$1" 2>/dev/null | awk '{for (i=1;i<=NF;i++) if ($i=="dev") {print $(i+1); exit}}'
}

route_via() {
  ip route get "$1" 2>/dev/null | awk '{for (i=1;i<=NF;i++) if ($i=="via") {print $(i+1); exit}}'
}

load_1m() { awk '{print $1}' /proc/loadavg 2>/dev/null || echo n/a; }

cpu_busy_pct() {
  awk '/^cpu /{idle=$5+$6;t=0;for(i=2;i<=NF;i++)t+=$i;if(t>0)printf"%.0f",100*(t-idle)/t}' /proc/stat 2>/dev/null || echo n/a
}

conntrack_kv() {
  _c="$(cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null)" || _c=""
  _m="$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null)" || _m=""
  if [ -n "$_c" ] && [ -n "$_m" ] && [ "$_m" -gt 0 ] 2>/dev/null; then
    printf 'conntrack=%s/%s(%s%%)' "$_c" "$_m" "$((_c * 100 / _m))"
  fi
}

neigh_state() { ip neigh show "$1" 2>/dev/null | awk '{print $6; exit}'; }

num_ge() {
  awk -v a="$1" -v b="$2" 'BEGIN{exit(a+0>=b+0)?0:1}' 2>/dev/null
}

ping_stats() {
  _host="$1"; _cnt="${2:-$PING_COUNT}"; _dead="${3:-$PING_DEADLINE_S}"
  _out="$(ping -c "$_cnt" -w "$_dead" "$_host" 2>/dev/null)" || _out=""
  if [ -z "$_out" ]; then printf '%s %s %s %s %s' -1 -1 -1 -1 -1; return; fi
  _loss="$(printf '%s\n' "$_out" | awk '/packet loss/{for(i=1;i<=NF;i++) if($i~/%$/){gsub(/%/,"",$i);print $i;exit}}')"
  _rtt="$(printf '%s\n' "$_out" | sed -n 's/.*= \([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\) ms.*/\1 \2 \3 \4/p' | head -1)"
  if [ -n "$_rtt" ]; then
    set -- $_rtt
    printf '%s %s %s %s %s' "${_loss:-0}" "$1" "$2" "$3" "$4"
  else
    _avg="$(printf '%s\n' "$_out" | awk -F'[=/ ]' '/min\/avg\/max/{print $8;exit}')"
    printf '%s %s %s %s %s' "${_loss:-0}" "${_avg:--1}" "${_avg:--1}" "${_avg:--1}" 0
  fi
}

append_ping() {
  _line="$1"; _pfx="$2"; _host="$3"
  set -- $(ping_stats "$_host")
  _loss="$1"; _min="$2"; _avg="$3"; _max="$4"; _mdev="$5"
  printf '%s %s=%s %s_loss=%s%% %s_min_ms=%s %s_avg_ms=%s %s_max_ms=%s %s_mdev_ms=%s' \
    "$_line" "$_pfx" "$_host" "$_pfx" "$_loss" "$_pfx" "$_min" "$_pfx" "$_avg" "$_pfx" "$_max" "$_pfx" "$_mdev"
}

field_val() {
  printf '%s\n' "$2" | awk -v f="$1" '{for(i=1;i<=NF;i++) if($i~"^"f"="){split($i,a,"=");gsub(/%/,"",a[2]);print a[2];exit}}'
}

acquire_lock() {
  command -v flock >/dev/null 2>&1 || return 0
  _lock="$WAN_WATCH_LOCK"
  _fb="${TMPDIR:-/tmp}/wan-watch.$$.lock"
  _dir="$(dirname "$_lock")"
  if [ ! -w "$_dir" ] 2>/dev/null; then _lock="${TMPDIR:-/tmp}/wan-watch.$(id -u 2>/dev/null || echo 0).lock"; fi
  exec 9>"$_lock" 2>/dev/null || return 0
  flock -n 9 || { echo "wan-watch: skip (running)" >&2; exit 0; }
}

cooldown_active() {
  [ -r "$WAN_WATCH_COOLDOWN_FILE" ] || return 1
  _l="$(cat "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null)" || return 1
  _n="$(date +%s 2>/dev/null)" || return 1
  [ "$((_n - _l))" -lt "$WAN_WATCH_COOLDOWN_S" ] 2>/dev/null
}

# ---------------------------------------------------------------------------
# Probes: tunnels, services, kernel — cheap, every run
# ---------------------------------------------------------------------------
probe_emit() {
  _ts="$1"; _role="$2"; _host="$3"; _section="$4"
  shift 4
  emit probe "$_ts" "$_role" "$_host" "section=$_section $*"
}

probe_all() {
  _ts="$1"; _role="$2"; _host="$3"

  # Kernel modules (GRE/PPTP)
  if [ -r /proc/modules ]; then
    _gre="$(grep -E '^ip_gre |^gre ' /proc/modules 2>/dev/null | awk '{print $1}' | tr '\n' ',' | sed 's/,$//')"
    _pptp="$(grep -E 'nf_conntrack_pptp|nf_nat_pptp' /proc/modules 2>/dev/null | awk '{print $1}' | tr '\n' ',' | sed 's/,$//')"
    [ -n "$_gre" ] && probe_emit "$_ts" "$_role" "$_host" gre "loaded=${_gre}"
    [ -n "$_pptp" ] && probe_emit "$_ts" "$_role" "$_host" pptp_mod "loaded=${_pptp}"
  fi

  # GRE / gretap links
  ip -d link show type gre 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" '
    /^[0-9]+:/{name=$2; gsub(":","",name); state=$0; getline; detail=$0;
      printf "kind=probe ts=%s ver=3 role=%s host=%s section=gre_iface name=%s up=%s\n", ts,role,host,name,(index(state,"UP")?"yes":"no")}
  ' >> "$WAN_WATCH_LOG" 2>/dev/null || true

  # PPTP listeners / sessions
  _p1723="$(ss -lnpt 2>/dev/null | grep -c ':1723 ' || echo 0)"
  _pppd="$(pgrep -c pppd 2>/dev/null || echo 0)"
  _pptpd="$(pgrep -c pptpd 2>/dev/null || echo 0)"
  probe_emit "$_ts" "$_role" "$_host" pptp "listen1723=${_p1723} pppd=${_pppd} pptpd=${_pptpd}"

  # tinc
  _tinc="$(pgrep -c tincd 2>/dev/null || echo 0)"
  if [ "$_tinc" -gt 0 ] 2>/dev/null; then
    _tn=""
    if command -v tincctl >/dev/null 2>&1; then
      for _net in /etc/tinc/*/tinc.conf; do
        [ -f "$_net" ] || continue
        _n="$(basename "$(dirname "$_net")")"
        _st="$(tincctl -n "$_n" status 2>/dev/null | tr '\n' ' ' | sed 's/  */ /g')"
        probe_emit "$_ts" "$_role" "$_host" tinc "net=${_n} tincd=${_tinc} status=${_st:-unknown}"
      done
    else
      probe_emit "$_ts" "$_role" "$_host" tinc "tincd=${_tinc} status=no_tincctl"
    fi
  else
    probe_emit "$_ts" "$_role" "$_host" tinc "tincd=0"
  fi

  # WireGuard
  if command -v wg >/dev/null 2>&1; then
    wg show all 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" '
      /^interface:/ { iface=$2; peer=""; hs=""; ep=""; tx=""; rx="" }
      /^  public key:/ { pk=$3 }
      /^  endpoint:/ { ep=$2 }
      /^  latest handshake:/ { hs=$0; sub(/^  latest handshake: /,"",hs) }
      /^  transfer:/ { tx=$2; rx=$4 }
      /^$/ {
        if (iface != "")
          printf "kind=probe ts=%s ver=3 role=%s host=%s section=wireguard iface=%s endpoint=%s handshake=%s tx=%s rx=%s\n",
            ts, role, host, iface, ep, hs, tx, rx
      }
      END {
        if (iface != "")
          printf "kind=probe ts=%s ver=3 role=%s host=%s section=wireguard iface=%s endpoint=%s handshake=%s tx=%s rx=%s\n",
            ts, role, host, iface, ep, hs, tx, rx
      }
    ' >> "$WAN_WATCH_LOG" 2>/dev/null || true
  fi

  # iodine / dns tunnel iface
  _iod="$(pgrep -c iodined 2>/dev/null || echo 0)"
  probe_emit "$_ts" "$_role" "$_host" iodine "iodined=${_iod}"
  for _d in dns0 tun0 tap0; do
    ip link show "$_d" >/dev/null 2>&1 || continue
    ip -s link show "$_d" 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v d="$_d" '
      /RX:/{getline; re=$3; rd=$4} /TX:/{getline; te=$3; td=$4}
      END{printf "kind=probe ts=%s ver=3 role=%s host=%s section=tunnel_iface dev=%s rx_err=%s rx_drop=%s tx_err=%s tx_drop=%s\n",
        ts,role,host,d,re+0,rd+0,te+0,td+0}
    ' >> "$WAN_WATCH_LOG" 2>/dev/null || true
  done

  # Named / squid / nginx (edge services)
  for _svc in named bind9 squid nginx iodined pptpd; do
    _c="$(pgrep -c "$_svc" 2>/dev/null || echo 0)"
    [ "$_c" -gt 0 ] 2>/dev/null || continue
    _cpu="$(ps -eo pcpu,comm 2>/dev/null | awk -v s="$_svc" '$2~s{print $1; exit}')"
    probe_emit "$_ts" "$_role" "$_host" service "name=${_svc} procs=${_c} cpu=${_cpu:-n/a}"
  done

  # Key ifaces stats
  for _d in xenbr0 eno0 eth0 wlan0 br0; do
    ip link show "$_d" >/dev/null 2>&1 || continue
    ip -s link show "$_d" 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v d="$_d" '
      /RX:/{getline; re=$3; rd=$4} /TX:/{getline; te=$3; td=$4}
      END{printf "kind=probe ts=%s ver=3 role=%s host=%s section=iface dev=%s rx_drop=%s tx_drop=%s rx_err=%s tx_err=%s\n",
        ts,role,host,d,rd+0,td+0,re+0,te+0}
    ' >> "$WAN_WATCH_LOG" 2>/dev/null || true
    tc -s qdisc show dev "$_d" 2>/dev/null | head -3 | while IFS= read -r _ql; do
      probe_emit "$_ts" "$_role" "$_host" qdisc "dev=${_d} detail=${_ql}"
    done
  done

  # softnet drops
  if [ -r /proc/net/softnet_stat ]; then
    awk -v ts="$_ts" -v role="$_role" -v host="$_host" '$2+0>0{
      printf "kind=probe ts=%s ver=3 role=%s host=%s section=softnet cpu=%d dropped=%s\n",ts,role,host,NR-1,$2
    }' /proc/net/softnet_stat >> "$WAN_WATCH_LOG" 2>/dev/null || true
  fi

  # Routes + neigh for key targets
  for _label in wan:$WAN_PING_TARGET fe:$FE_PING_TARGET gw:${GW_PING_TARGET:-auto}; do
    _l="${_label%%:*}"; _t="${_label#*:}"
    [ "$_t" = "auto" ] && _t="$(default_gw)"
    [ -n "$_t" ] && [ "$_t" != none ] || continue
    ip route get "$_t" 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v l="$_l" -v t="$_t" '
      {dev=via=src=""; for(i=1;i<=NF;i++){if($i=="dev")dev=$(i+1);if($i=="via")via=$(i+1);if($i=="src")src=$(i+1)}
       printf "kind=probe ts=%s ver=3 role=%s host=%s section=route label=%s target=%s dev=%s via=%s src=%s\n",ts,role,host,l,t,dev,via,src}
    ' >> "$WAN_WATCH_LOG" 2>/dev/null || true
  done
  _ns="$(neigh_state "$FE_PING_TARGET")"
  [ -n "$_ns" ] && probe_emit "$_ts" "$_role" "$_host" neigh "ip=${FE_PING_TARGET} state=${_ns}"

  # iptables hot rules (router/fe)
  if command -v iptables >/dev/null 2>&1 && [ "$_role" = "router" ]; then
    for _spec in filter:FORWARD nat:PREROUTING nat:POSTROUTING; do
      _tbl="${_spec%%:*}"; _ch="${_spec#*:}"
      iptables -t "$_tbl" -L "$_ch" -n -v --line-numbers 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v tbl="$_tbl" -v ch="$_ch" '
        /^[0-9]+/ && $2+0>1000 {
          printf "kind=probe ts=%s ver=3 role=%s host=%s section=iptables tbl=%s chain=%s line=%s pkts=%s target=%s\n",
            ts,role,host,tbl,ch,$1,$2,$4
        }' | head -6 >> "$WAN_WATCH_LOG" 2>/dev/null || true
    done
  fi

  # Top CPU
  ps -eo pcpu=,pmem=,comm= --sort=-pcpu 2>/dev/null | head -5 | awk -v ts="$_ts" -v role="$_role" -v host="$_host" '
    NF>=3{printf "kind=probe ts=%s ver=3 role=%s host=%s section=proc rank=%d cpu=%s mem=%s comm=%s\n",ts,role,host,NR,$1,$2,$3}
  ' >> "$WAN_WATCH_LOG" 2>/dev/null || true

  ss -s 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" '/TCP:|UDP:/{
    gsub(/^[ \t]+/,""); printf "kind=probe ts=%s ver=3 role=%s host=%s section=ss %s\n",ts,role,host,$0
  }' >> "$WAN_WATCH_LOG" 2>/dev/null || true
}

# ---------------------------------------------------------------------------
# Deep capture (on anomaly)
# ---------------------------------------------------------------------------
deep_burst_line() {
  _ts="$1"; _role="$2"; _host="$3"; _label="$4"; _target="$5"
  set -- $(ping_stats "$_target" "$DEEP_PING_COUNT" "$DEEP_PING_DEADLINE_S")
  emit deep "$_ts" "$_role" "$_host" \
    "parent_ts=$_ts section=burst label=$_label target=$_target loss=$1% min_ms=$2 avg_ms=$3 max_ms=$4 mdev_ms=$5"
}

deep_capture() {
  _ts="$1"; _role="$2"; _host="$3"; _triggers="$4"
  emit alert "$_ts" "$_role" "$_host" "parent_ts=$_ts triggers=\"$_triggers\""
  _ct="$(conntrack_kv)"
  emit deep "$_ts" "$_role" "$_host" "parent_ts=$_ts section=host load1=$(load_1m) cpu_busy=$(cpu_busy_pct)% mem_avail_kb=$(awk '/^MemAvailable:/{print $2}' /proc/meminfo 2>/dev/null || echo n/a) ${_ct}"
  deep_burst_line "$_ts" "$_role" "$_host" wan "$WAN_PING_TARGET"
  deep_burst_line "$_ts" "$_role" "$_host" fe "$FE_PING_TARGET"
  _gw="$GW_PING_TARGET"; [ "$_gw" = auto ] && _gw="$(default_gw)"
  [ -n "$_gw" ] && [ "$_gw" != none ] && deep_burst_line "$_ts" "$_role" "$_host" gw "$_gw"
  probe_all "$_ts" "$_role" "$_host"
}

detect_triggers() {
  _line="$1"; _role="$2"; _t=""
  for _p in wan fe be gw router; do
    _a="$(field_val "${_p}_avg_ms" "$_line")"
    _l="$(field_val "${_p}_loss" "$_line")"
    _m="$(field_val "${_p}_mdev_ms" "$_line")"
    _thr="$FE_ALERT_MS"
    case "$_p" in wan) _thr="$WAN_ALERT_MS";; fe) _thr="$FE_ALERT_MS";;
      be) _thr="$BE_ALERT_MS";; gw) _thr="$GW_ALERT_MS";;
      router) _thr="$ROUTER_ALERT_MS";; esac
    num_ge "$_a" "$_thr" && _t="${_t}${_t:+,}${_p}_avg=${_a}"
    num_ge "$_l" "$LOSS_ALERT_PCT" && _t="${_t}${_t:+,}${_p}_loss=${_l}"
    num_ge "$_m" "$MDEV_ALERT_MS" && _t="${_t}${_t:+,}${_p}_mdev=${_m}"
  done
  _ct="$(printf '%s\n' "$_line" | sed -n 's/.*(\([0-9]*\)%).*/\1/p' | head -1)"
  num_ge "$_ct" "$CONNTRACK_ALERT_PCT" && _t="${_t}${_t:+,}conntrack_pct=${_ct}"
  _wan="$(field_val wan_avg_ms "$_line")"; _fe="$(field_val fe_avg_ms "$_line")"
  if [ "$_role" = pi ] && num_ge "$_fe" "$FE_ALERT_MS" && ! num_ge "$_wan" "$WAN_ALERT_MS"; then
    case "$_t" in *fe_avg=*) ;; *) _t="${_t}${_t:+,}pi_fe_path";; esac
  fi
  if [ "$_role" = router ] && num_ge "$_wan" "$WAN_ALERT_MS" && ! num_ge "$_fe" "$FE_ALERT_MS"; then
    case "$_t" in *wan_avg=*) ;; *) _t="${_t}${_t:+,}router_wan_path";; esac
  fi
  printf '%s' "$_t"
}

# ---------------------------------------------------------------------------
# Verdict engine — runs EVERY time on current snapshot
# ---------------------------------------------------------------------------
compute_verdict() {
  _line="$1"; _role="$2"
  _probe_hint="${3:-}"
  _wan="$(field_val wan_avg_ms "$_line")"
  _fe="$(field_val fe_avg_ms "$_line")"
  _be="$(field_val be_avg_ms "$_line")"
  _gw="$(field_val gw_avg_ms "$_line")"
  _wmdev="$(field_val wan_mdev_ms "$_line")"
  _ct="$(printf '%s\n' "$_line" | sed -n 's/.*(\([0-9]*\)%).*/\1/p' | head -1)"

  _verdict="OK"
  _conf="high"
  _summary="All probes within thresholds."
  _action="none"

  if [ "$_role" = router ]; then
    if num_ge "$_wan" "$WAN_ALERT_MS" && ! num_ge "$_fe" 5; then
      _verdict="MONSTRO_WAN_BLOAT"
      _summary="WAN leg slow (~${_wan}ms) but FE local OK (~${_fe}ms). Edge xenbr0/upstream queue."
      _action="SQM/fq_codel on xenbr0; check gw $(route_via "$WAN_PING_TARGET" 2>/dev/null || echo 10.7.16.228); review squid/named load"
    elif num_ge "$_gw" "$GW_ALERT_MS" && num_ge "$_wan" "$WAN_ALERT_MS"; then
      _verdict="UPSTREAM_GW"
      _summary="WAN (${_wan}ms) and gw (${_gw}ms) both high — parent hop 10.7.16.228 stressed."
      _action="Check upstream link; correlate with ISP/gateway admin"
    elif num_ge "$_ct" "$CONNTRACK_ALERT_PCT"; then
      _verdict="CONNTRACK_PRESSURE"
      _summary="Conntrack at ${_ct}% — new flows may stall."
      _action="Inspect conntrack count; reduce NAT/tunnel sessions or raise nf_conntrack_max"
    elif num_ge "$_wmdev" "$MDEV_ALERT_MS"; then
      _verdict="WAN_JITTER"
      _summary="WAN jitter high (mdev=${_wmdev}ms) avg=${_wan}ms."
      _action="Bufferbloat likely; enable fq_codel"
    fi
  elif [ "$_role" = pi ]; then
    if num_ge "$_fe" "$FE_ALERT_MS" && ! num_ge "$_wan" "$WAN_ALERT_MS"; then
      _verdict="PATH_TO_FE"
      _summary="Pi ISP OK (~${_wan}ms) but FE path slow (~${_fe}ms). Problem at monstro edge, not Pi WiFi."
      _action="Check monstro wan-watch verdict; your traffic crosses modem WAN queue"
    elif num_ge "$_wan" "$WAN_ALERT_MS"; then
      _verdict="PI_ISP"
      _summary="Pi WAN to 1.1.1.1 slow (${_wan}ms)."
      _action="Local ISP/WiFi issue on Pi side"
    fi
  elif [ "$_role" = fe ]; then
    if num_ge "$_wan" "$WAN_ALERT_MS"; then
      _verdict="FE_WAN_EGRESS"
      _summary="FE WAN egress slow (${_wan}ms)."
      _action="Check FE default route and nginx upstream separately"
    fi
  fi

  # Probe hints from same run (grep log tail — set by caller in _probe_hint)
  case "${_probe_hint:-}" in
    *dns0*tx_drop=*|*tunnel_iface*dns0*)
      if [ "$_verdict" = "OK" ] || [ "$_verdict" = "MONSTRO_WAN_BLOAT" ]; then
        _verdict="TUNNEL_DNS0_LOAD"
        _summary="${_summary} iodine/dns0 tunnel drops elevated."
        _action="${_action}; inspect iodined and dns0 tx_drop"
        _conf="medium"
      fi ;;
    *named*cpu=*)
      if [ "$_verdict" = "MONSTRO_WAN_BLOAT" ] || [ "$_verdict" = "OK" ]; then
        _summary="${_summary} named DNS active on router."
        _conf="medium"
      fi ;;
  esac

  printf 'verdict=%s confidence=%s summary="%s" action="%s" wan_ms=%s fe_ms=%s gw_ms=%s' \
    "$_verdict" "$_conf" "$_summary" "$_action" "$_wan" "$_fe" "$_gw"
}

# ---------------------------------------------------------------------------
# Main cron run
# ---------------------------------------------------------------------------
run_watch() {
  command -v ping >/dev/null 2>&1 || { echo "wan-watch: need ping" >&2; exit 1; }
  acquire_lock

  _ts="$(date -Iseconds 2>/dev/null || date '+%Y-%m-%dT%H:%M:%S%z')"
  _role="$(detect_role)"
  _host="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)"

  _gw="$GW_PING_TARGET"; [ "$_gw" = auto ] && _gw="$(default_gw)"
  _ct="$(conntrack_kv)"
  _wdev="$(route_dev "$WAN_PING_TARGET")"; _fdev="$(route_dev "$FE_PING_TARGET")"
  _wgw="$(route_via "$WAN_PING_TARGET")"
  _ns="$(neigh_state "$FE_PING_TARGET")"

  _snap="$(append_ping "" wan "$WAN_PING_TARGET")"
  _snap="$(append_ping "$_snap" fe "$FE_PING_TARGET")"
  [ -n "$BE_PING_TARGET" ] && [ "$BE_PING_TARGET" != none ] && _snap="$(append_ping "$_snap" be "$BE_PING_TARGET")"
  [ -n "$ROUTER_PING_TARGET" ] && [ "$ROUTER_PING_TARGET" != none ] && _snap="$(append_ping "$_snap" router "$ROUTER_PING_TARGET")"
  [ -n "$_gw" ] && [ "$_gw" != none ] && _snap="$(append_ping "$_snap" gw "$_gw")"
  _snap="$_snap load1=$(load_1m) cpu_busy=$(cpu_busy_pct)%"
  [ -n "$_ct" ] && _snap="$_snap $_ct"
  [ -n "$_wdev" ] && _snap="$_snap route_wan=$_wdev"
  [ -n "$_fdev" ] && _snap="$_snap route_fe=$_fdev"
  [ -n "$_wgw" ] && _snap="$_snap via_wan=$_wgw"
  [ -n "$_ns" ] && _snap="$_snap neigh_fe=$_ns"

  emit snapshot "$_ts" "$_role" "$_host" "$_snap" >/dev/null

  probe_all "$_ts" "$_role" "$_host"

  _probe_hint="$(tail -30 "$WAN_WATCH_LOG" 2>/dev/null | grep 'kind=probe' | tr '\n' ' ')"
  _verdict_kv="$(compute_verdict "$_snap" "$_role" "$_probe_hint")"
  _verdict_line="kind=verdict ts=$_ts ver=3 role=$_role host=$_host $_verdict_kv"
  printf '%s\n' "$_verdict_line" >> "$WAN_WATCH_LOG"

  _triggers="$(detect_triggers "$_snap" "$_role")"
  _do_deep=0
  if [ -n "$_triggers" ]; then _do_deep=1; fi
  if [ "$WAN_WATCH_DEEP" = "always" ]; then _do_deep=1; fi
  if [ "$WAN_WATCH_DEEP" = "never" ]; then _do_deep=0; fi

  if [ "$_do_deep" = 1 ]; then
    if cooldown_active && [ "$WAN_WATCH_DEEP" != "always" ]; then
      emit alert "$_ts" "$_role" "$_host" "parent_ts=$_ts triggers=\"$_triggers\" action=skipped_cooldown" >/dev/null
    else
      deep_capture "$_ts" "$_role" "$_host" "$_triggers"
      date +%s > "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null || true
    fi
  fi

  # Human one-liner to cron log
  printf 'WAN-WATCH [%s/%s] %s\n' "$_role" "$_host" "$_verdict_kv"
}

# ---------------------------------------------------------------------------
# --analyze: final conclusion from log without waiting
# ---------------------------------------------------------------------------
analyze_log() {
  _log="$WAN_WATCH_LOG"
  _hours="${1:-$WAN_ANALYZE_HOURS}"
  [ -r "$_log" ] || { echo "wan-watch: no log $_log" >&2; exit 1; }

  _cut="$(date -d "${_hours} hours ago" '+%Y-%m-%dT%H:%M' 2>/dev/null || echo "")"

  echo "=== wan-watch analyze (last ${_hours}h) ==="
  echo "log=$_log"
  echo ""

  awk -v cut="$_cut" '
    function ts_ok(t) { return (cut == "" || t >= cut) }
    /^kind=snapshot / && ts_ok($0) {
      for(i=1;i<=NF;i++) {
        if($i~/^wan_avg_ms=/){split($i,a,"="); wa=a[2]+0; wn++}
        if($i~/^fe_avg_ms=/){split($i,a,"="); fa=a[2]+0; fn++}
        if($i~/^gw_avg_ms=/){split($i,a,"="); ga=a[2]+0; gn++}
        if($i~/^role=/){split($i,a,"="); role=a[2]}
      }
      if(wa>=50){wb++} else if(wa>=0){wg++}
      if(fa>=50){fb++} else if(fa>=0){fg++}
      if(wa>=50 && fa<5) rw_ok_fe++   # router pattern
      if(fa>=50 && wa<30) pi_bad++    # pi pattern
      sn++
    }
    /^kind=verdict / && ts_ok($0) {
      if($0~/verdict=OK/) okv++; else badv++
      for(i=1;i<=NF;i++) if($i~/^verdict=/){split($i,a,"="); vc[a[2]]++}
    }
    /^kind=probe / && /section=tunnel_iface/ && /tx_drop=[1-9]/ { dns_drop++ }
    /^kind=probe / && /section=service/ && /name=named/ { named++ }
    /^kind=probe / && /section=service/ && /name=squid/ { squid++ }
    END {
      printf "snapshots=%d (wan_bad=%d wan_good=%d fe_bad=%d fe_good=%d)\n", sn, wb+0, wg+0, fb+0, fg+0
      printf "router_wan_bad_fe_local_ok=%d (MONSTRO_WAN_BLOAT signature)\n", rw_ok_fe+0
      printf "pi_fe_bad_wan_ok=%d (PATH_TO_FE signature)\n", pi_bad+0
      printf "verdicts: ok=%d anomaly=%d\n", okv+0, badv+0
      printf "verdict_counts:"
      for(v in vc) printf " %s=%d", v, vc[v]
      printf "\n"
      printf "probe_hints: dns0_tx_drop_events=%d named_samples=%d squid_samples=%d\n", dns_drop+0, named+0, squid+0
      pct = (sn>0) ? int(100*(wb+fb)/(2*sn)) : 0
      printf "\n"
      if(rw_ok_fe>=3 || pi_bad>=3) {
        printf "CONCLUSION: MONSTRO WAN EDGE QUEUEING (confidence=HIGH)\n"
        printf "  Evidence: wan slow + fe local OK; Pi ISP stays ~9ms in prior data.\n"
        printf "  Not: FE VM, BE, Pi WiFi, global ISP.\n"
        printf "  Fix order: 1) fq_codel/SQM on xenbr0  2) gw 10.7.16.228  3) squid/named  4) iodine dns0\n"
      } else if(wb+fb==0) {
        printf "CONCLUSION: HEALTHY (confidence=HIGH)\n"
      } else {
        printf "CONCLUSION: MIXED/INTERMITTENT (%d%% bad samples) — keep */5 cron running\n", pct
      }
    }
  ' "$_log"

  echo ""
  echo "=== last 5 verdicts ==="
  grep '^kind=verdict ' "$_log" 2>/dev/null | tail -5 | while IFS= read -r _l; do
    _t="$(printf '%s\n' "$_l" | sed 's/.*ts=\([^ ]*\).*/\1/')"
    _v="$(printf '%s\n' "$_l" | sed 's/.*verdict=\([^ ]*\).*/\1/')"
    _s="$(printf '%s\n' "$_l" | sed 's/.*summary="\([^"]*\)".*/\1/')"
    printf '  %s  %s  %s\n' "$_t" "$_v" "$_s"
  done
}

timeline_log() {
  _log="$WAN_WATCH_LOG"
  [ -r "$_log" ] || { echo "no log"; exit 1; }
  echo "=== hourly (snapshot wan/fe bad if avg>=50) ==="
  awk '
    /^kind=snapshot / {
      ts=$0; sub(/.*ts=/,"",ts); sub(/ .*/,"",ts)
      hour=substr(ts,1,13)
      for(i=1;i<=NF;i++){
        if($i~/^wan_avg_ms=/){split($i,a,"="); w=a[2]+0}
        if($i~/^fe_avg_ms=/){split($i,a,"="); f=a[2]+0}
      }
      t[hour]++; if(w>=50||f>=50) b[hour]++
    }
    END {
      for(h in t){ bh=b[h]+0; printf "  %s  %d/%d bad  ", h, bh, t[h]
        for(i=0;i<t[h];i++) printf(i<bh?"#":".")
        printf "\n"
      }
    }
  ' "$_log" | sort
}

full_report() {
  _ts="$(date -Iseconds 2>/dev/null || date '+%Y-%m-%dT%H:%M:%S%z')"
  _role="$(detect_role)"
  _host="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)"
  echo "=== wan-watch FULL @ $_ts role=$_role host=$_host ==="
  PING_COUNT=5 PING_DEADLINE_S=10
  for _t in "$WAN_PING_TARGET" "$FE_PING_TARGET" "$BE_PING_TARGET" "$ROUTER_PING_TARGET" "$(default_gw)"; do
    [ -n "$_t" ] && [ "$_t" != none ] || continue
    echo "--- ping $_t ---"
    ping -c 5 -w 10 "$_t" 2>/dev/null || echo "fail"
  done
  echo "--- host ---"
  echo "load=$(load_1m) cpu=$(cpu_busy_pct)% $(conntrack_kv)"
  echo "--- probes ---"
  probe_all "$_ts" "$_role" "$_host"
  tail -40 "$WAN_WATCH_LOG" | grep "^kind=probe "
  echo "--- verdict ---"
  _snap="ts=$_ts role=$_role host=$_host"
  _snap="$(append_ping "$_snap" wan "$WAN_PING_TARGET")"
  _snap="$(append_ping "$_snap" fe "$FE_PING_TARGET")"
  compute_verdict "$_snap" "$_role"
  echo ""
}

correlate_logs() {
  if [ "$#" -lt 1 ]; then echo "wan-watch: --correlate needs log files" >&2; exit 1; fi
  _tmp="$(mktemp "${TMPDIR:-/tmp}/wan-corr.XXXXXX")"
  trap 'rm -f "$_tmp"' EXIT INT HUP
  for _path in "$@"; do
    [ -r "$_path" ] || continue
    while IFS= read -r _line || [ -n "$_line" ]; do
      case "$_line" in
        kind=snapshot*) _ts="${_line#*ts=}"; _ts="${_ts%% *}" ;;
        ts=*) _ts="${_line#ts=}"; _ts="${_ts%% *}" ;;
        *) continue ;;
      esac
      _date="${_ts%%T*}"; _rest="${_ts#*T}"; _hour="${_rest%%:*}"
      _min="${_rest#*:}"; _min="${_min%%:*}"; _min="${_min%%+*}"; _min="${_min%%-*}"
      _slot="${_date}T${_hour}:$(printf '%02d' $((_min / 15 * 15)))"
      _role="$(printf '%s\n' "$_line" | awk '{for(i=1;i<=NF;i++) if($i~/^role=/){split($i,a,"=");print a[2];exit}}')"
      [ -n "$_role" ] || _role="$(basename "$_path" .log)"
      printf '%s\t%s\t%s\t%s\n' "$_slot" "$_role" "$_ts" "$_line" >> "$_tmp"
    done < "$_path"
  done
  printf 'slot\t'
  _roles="$(awk -F'\t' '{print $2}' "$_tmp" | sort -u | tr '\n' ' ')"
  for _r in $_roles; do printf '%s_wan\t%s_fe\t%s_verdict\t' "$_r" "$_r" "$_r"; done
  printf '\n'
  awk -F'\t' '
    {
      slot=$1; role=$2; ts=$3; line=$4; key=slot SUBSEP role
      if (!(key in best) || ts>best[key]) {
        best[key]=ts; wan=fe="-"; ver="-"
        for(i=1;i<=split(line,a," ");i++){
          if(a[i]~/^wan_avg_ms=/){split(a[i],b,"=");wan=b[2]}
          if(a[i]~/^fe_avg_ms=/){split(a[i],b,"=");fe=b[2]}
        }
        W[slot,role,"wan"]=wan; W[slot,role,"fe"]=fe
        slots[slot]=1; roles[role]=1
      }
    }
    END{
      nr=0; for(r in roles) rl[++nr]=r; asort(rl)
      ns=0; for(s in slots) sl[++ns]=s; asort(sl)
      for(i=1;i<=ns;i++){
        s=sl[i]; printf "%s", s
        for(j=1;j<=nr;j++){
          r=rl[j]; printf "\t%s\t%s\t-", W[s,r,"wan"]+0?W[s,r,"wan"]:"-", W[s,r,"fe"]+0?W[s,r,"fe"]:"-"
        }
        printf "\n"
      }
    }
  ' "$_tmp"
}

usage() {
  cat <<EOF
wan-watch.sh — unified network watchdog (ver=3)

Cron:  */5 * * * * root . /etc/conf.d/wan-watch; /usr/local/bin/wan-watch.sh

Commands:
  (default)     snapshot + probes + verdict (+ deep if bad)
  --full        immediate human report
  --analyze [H] conclusion from log (default ${WAN_ANALYZE_HOURS}h)
  --timeline    hourly bad/good bars
  --force-deep  set WAN_WATCH_DEEP=always for one run
  --tail [N]    last N snapshot+verdict lines
  --correlate FILE…  merge Pi+router logs by 15-min slot
  --help

Log: $WAN_WATCH_LOG (kind=snapshot|probe|verdict|alert|deep)
EOF
}

case "${1:-}" in
  -h|--help|help) usage ;;
  --full|full) full_report ;;
  --analyze|analyze) analyze_log "${2:-$WAN_ANALYZE_HOURS}" ;;
  --timeline|timeline) timeline_log ;;
  --force-deep|force-deep) WAN_WATCH_DEEP=always run_watch ;;
  --tail|tail)
    _n="${2:-20}"
    grep -E '^kind=(snapshot|verdict|alert) ' "$WAN_WATCH_LOG" 2>/dev/null | tail -n "$_n" || true
    ;;
  "") run_watch ;;
  *) echo "wan-watch: unknown $1"; usage; exit 1 ;;
esac
