bumping

Net topology: sirene (P2P), spider (MCU), spider net (SFU), pautina (net, mesh), hydra (hybrid), code names
2026-07-29 03:38:52 +03:00 · 2026-06-12 22:13:28 +02:00
parent ad35a60e6c
commit 9acbdbc433
4 changed files with 564 additions and 40 deletions
--- a/examples/network/README.md
+++ b/examples/network/README.md
@@ -2,6 +2,52 @@

 Scripts for **router**, **FE**, and **external vantage** (e.g. Raspberry Pi) — not BE app hosts.

+## Auto watchdog (v2.1)
+
+When a cron snapshot exceeds thresholds, `wan-snapshot.sh` appends **alert** + **deep** lines to the same log (`WAN_WATCH_LOG`).
+
+| `kind=` | Meaning |
+|---------|---------|
+| `snapshot` | Normal 15-min ping row |
+| `alert` | Threshold breached; lists `triggers=` |
+| `deep` | One parseable fact (`section=host`, `burst`, `iface`, `iptables`, …) |
+
+**Deploy both files:**
+
+```bash
+sudo cp examples/network/wan-snapshot.sh examples/network/wan-deep-diag.sh /usr/local/bin/
+sudo chmod +x /usr/local/bin/wan-snapshot.sh /usr/local/bin/wan-deep-diag.sh
+```
+
+**Env** (`/etc/conf.d/wan-snapshot`):
+
+```sh
+WAN_WATCH_LOG=/var/log/wan-watch.log
+WAN_SNAPSHOT_LOG=/var/log/wan-watch.log
+WAN_SNAPSHOT_WATCH=1
+```
+
+**Triggers (defaults):** `*_avg_ms >= 50`, `*_loss >= 1%`, `mdev >= 80`, `conntrack >= 80%`, `neigh_fe` not REACHABLE/DELAY, plus role hints `pi_fe_path` / `router_wan_path`.
+
+**Cooldown:** one deep capture per 14 min (`WAN_WATCH_COOLDOWN_S=840`) to avoid cron pile-up.
+
+**Review after a bad spell:**
+
+```bash
+/usr/local/bin/wan-snapshot.sh --alerts 80
+/usr/local/bin/wan-snapshot.sh --correlate /var/log/wan-watch.log …
+grep 'kind=deep.*section=burst' /var/log/wan-watch.log | tail
+grep 'kind=deep.*section=iptables' /var/log/wan-watch.log | tail
+```
+
+Example deep lines:
+
+```text
+kind=alert ts=2026-06-12T21:45:01+03:00 role=router host=monstro parent_ts=... triggers="wan_avg=535.365,router_wan_path"
+kind=deep ts=2026-06-12T21:45:01+03:00 role=router host=monstro parent_ts=... section=burst label=wan target=1.1.1.1 loss=0% avg_ms=520 max_ms=790 mdev_ms=110
+kind=deep ts=2026-06-12T21:45:01+03:00 role=router host=monstro parent_ts=... section=iptables tbl=filter chain=FORWARD line=3 pkts=188000 target=ACCEPT
+```
+
 ## `wan-snapshot.sh` (v2)

 Periodic ping comparison plus lightweight host/path context.
--- a/examples/network/wan-deep-diag.sh
+++ b/examples/network/wan-deep-diag.sh
@@ -0,0 +1,287 @@
+# wan-deep-diag.sh — parseable deep diagnostics (sourced by wan-snapshot.sh).
+# Not meant to be executed directly.
+#
+# Emits kind=deep lines to WAN_WATCH_LOG (key=value, one fact per line).
+
+wan_deep_load_1m() {
+  awk '{print $1}' /proc/loadavg 2>/dev/null || echo "n/a"
+}
+
+wan_deep_cpu_busy_pct() {
+  awk '/^cpu / {
+    idle = $5 + $6; total = 0
+    for (i = 2; i <= NF; i++) total += $i
+    if (total > 0) printf "%.0f", 100 * (total - idle) / total
+  }' /proc/stat 2>/dev/null || echo "n/a"
+}
+
+wan_deep_mem_avail_kb() {
+  awk '/^MemAvailable:/ { print $2 }' /proc/meminfo 2>/dev/null || echo "n/a"
+}
+
+wan_deep_conntrack_fields() {
+  _cur="$(cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null)" || _cur=""
+  _max="$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null)" || _max=""
+  if [ -n "$_cur" ] && [ -n "$_max" ] && [ "$_max" -gt 0 ] 2>/dev/null; then
+    _pct=$((_cur * 100 / _max))
+    printf 'conntrack_cur=%s conntrack_max=%s conntrack_pct=%s' "$_cur" "$_max" "$_pct"
+  else
+    printf 'conntrack_cur=n/a conntrack_max=n/a conntrack_pct=n/a'
+  fi
+}
+
+wan_deep_emit() {
+  # wan_deep_emit KIND TS ROLE HOST [key=value ...]
+  _kind="$1"
+  _ts="$2"
+  _role="$3"
+  _host="$4"
+  shift 4
+  _line="kind=$_kind ts=$_ts role=$_role host=$_host $*"
+  _dir="$(dirname "$WAN_WATCH_LOG")"
+  [ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true
+  printf '%s\n' "$_line" >> "$WAN_WATCH_LOG"
+}
+
+# Burst ping (fast) -> one kind=deep line.
+wan_deep_burst_ping() {
+  _parent_ts="$1"
+  _role="$2"
+  _hostn="$3"
+  _label="$4"
+  _target="$5"
+  _count="${6:-5}"
+  _deadline="${7:-6}"
+
+  _out="$(ping -c "$_count" -w "$_deadline" "$_target" 2>/dev/null)" || _out=""
+  if [ -z "$_out" ]; then
+    wan_deep_emit deep "$_parent_ts" "$_role" "$_hostn" \
+      "parent_ts=$_parent_ts section=burst label=$_label target=$_target loss=-1 avg_ms=-1"
+    return 0
+  fi
+
+  _loss="$(printf '%s\n' "$_out" | awk '/packet loss/ {
+    for (i = 1; i <= NF; i++) if ($i ~ /%$/) { gsub(/%/, "", $i); print $i; exit }
+  }')"
+  _rtt="$(printf '%s\n' "$_out" | sed -n 's/.*= \([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\) ms.*/\1 \2 \3 \4/p' | head -1)"
+  _min="-1" _avg="-1" _max="-1" _mdev="-1"
+  if [ -n "$_rtt" ]; then
+    _min="${_rtt%% *}"
+    _rest="${_rtt#* }"
+    _avg="${_rest%% *}"
+    _rest="${_rest#* }"
+    _max="${_rest%% *}"
+    _mdev="${_rest#* }"
+  fi
+  [ -n "$_loss" ] || _loss="-1"
+
+  wan_deep_emit deep "$_parent_ts" "$_role" "$_hostn" \
+    "parent_ts=$_parent_ts section=burst label=$_label target=$_target loss=${_loss}% min_ms=$_min avg_ms=$_avg max_ms=$_max mdev_ms=$_mdev"
+}
+
+wan_deep_host_health() {
+  _parent_ts="$1"
+  _role="$2"
+  _hostn="$3"
+  _ct="$(wan_deep_conntrack_fields)"
+  wan_deep_emit deep "$_parent_ts" "$_role" "$_hostn" \
+    "parent_ts=$_parent_ts section=host load1=$(wan_deep_load_1m) cpu_busy=$(wan_deep_cpu_busy_pct)% mem_avail_kb=$(wan_deep_mem_avail_kb) $_ct"
+}
+
+wan_deep_top_procs() {
+  _parent_ts="$1"
+  _role="$2"
+  _hostn="$3"
+  _n="${4:-5}"
+  ps -eo pcpu=,pmem=,comm= --sort=-pcpu 2>/dev/null | head -n "$_n" | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v n="$_n" '
+    NF >= 3 {
+      rank = NR
+      printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=proc rank=%d cpu=%s mem=%s comm=%s\n",
+        ts, role, host, ts, rank, $1, $2, $3
+    }
+  ' >> "$WAN_WATCH_LOG" 2>/dev/null || true
+}
+
+wan_deep_iface_stats() {
+  _parent_ts="$1"
+  _role="$2"
+  _hostn="$3"
+  _dev="$4"
+  [ -n "$_dev" ] || return 0
+  ip -s link show "$_dev" 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v dev="$_dev" '
+    /^[0-9]+:/ { rx_b = tx_b = rx_p = tx_p = rx_drop = tx_drop = rx_err = tx_err = 0 }
+    /RX: bytes/ { rx_b = $3; rx_p = $2 }
+    /RX:.*packets/ { getline; rx_err = $3; rx_drop = $4 }
+    /TX: bytes/ { tx_b = $3; tx_p = $2 }
+    /TX:.*packets/ { getline; tx_err = $3; tx_drop = $4 }
+    END {
+      printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=iface dev=%s rx_bytes=%s tx_bytes=%s rx_drop=%s tx_drop=%s rx_err=%s tx_err=%s\n",
+        ts, role, host, ts, dev, rx_b, tx_b, rx_drop, tx_drop, rx_err, tx_err
+    }
+  ' >> "$WAN_WATCH_LOG" 2>/dev/null || true
+}
+
+wan_deep_all_ifaces_nonzero() {
+  _parent_ts="$1"
+  _role="$2"
+  _hostn="$3"
+  ip -s link 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" '
+    /^[0-9]+:/ {
+      if (iface != "" && iface !~ /^lo$/ && (rx_drop + tx_drop + rx_err + tx_err) > 0)
+        printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=iface dev=%s rx_drop=%s tx_drop=%s rx_err=%s tx_err=%s\n",
+          ts, role, host, ts, iface, rx_drop, tx_drop, rx_err, tx_err
+      iface = $2; gsub(":", "", iface)
+      rx_drop = tx_drop = rx_err = tx_err = 0
+    }
+    /RX:/ { getline; rx_err = $3; rx_drop = $4 }
+    /TX:/ { getline; tx_err = $3; tx_drop = $4 }
+    END {
+      if (iface != "" && iface !~ /^lo$/ && (rx_drop + tx_drop + rx_err + tx_err) > 0)
+        printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=iface dev=%s rx_drop=%s tx_drop=%s rx_err=%s tx_err=%s\n",
+          ts, role, host, ts, iface, rx_drop, tx_drop, rx_err, tx_err
+    }
+  ' >> "$WAN_WATCH_LOG" 2>/dev/null || true
+}
+
+wan_deep_softnet() {
+  _parent_ts="$1"
+  _role="$2"
+  _hostn="$3"
+  [ -r /proc/net/softnet_stat ] || return 0
+  awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" '
+    $2 + 0 > 0 {
+      printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=softnet cpu=%d dropped=%s processed=%s\n",
+        ts, role, host, ts, NR - 1, $2, $1
+    }
+  ' /proc/net/softnet_stat >> "$WAN_WATCH_LOG" 2>/dev/null || true
+}
+
+wan_deep_neigh() {
+  _parent_ts="$1"
+  _role="$2"
+  _hostn="$3"
+  _ip="$4"
+  [ -n "$_ip" ] || return 0
+  ip neigh show "$_ip" 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v ip="$_ip" '
+    NF >= 5 {
+      printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=neigh ip=%s lladdr=%s dev=%s state=%s\n",
+        ts, role, host, ts, ip, $5, $3, $6
+    }
+  ' >> "$WAN_WATCH_LOG" 2>/dev/null || true
+}
+
+wan_deep_route() {
+  _parent_ts="$1"
+  _role="$2"
+  _hostn="$3"
+  _label="$4"
+  _target="$5"
+  [ -n "$_target" ] || return 0
+  ip route get "$_target" 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v label="$_label" -v target="$_target" '
+    {
+      dev = via = ""; src = ""
+      for (i = 1; i <= NF; i++) {
+        if ($i == "dev") dev = $(i + 1)
+        if ($i == "via") via = $(i + 1)
+        if ($i == "src") src = $(i + 1)
+      }
+      printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=route label=%s target=%s dev=%s via=%s src=%s\n",
+        ts, role, host, ts, label, target, dev, via, src
+    }
+  ' >> "$WAN_WATCH_LOG" 2>/dev/null || true
+}
+
+wan_deep_iptables_hot() {
+  _parent_ts="$1"
+  _role="$2"
+  _hostn="$3"
+  command -v iptables >/dev/null 2>&1 || return 0
+  for _spec in "filter:FORWARD" "nat:PREROUTING" "nat:POSTROUTING"; do
+    _tbl="${_spec%%:*}"
+    _chain="${_spec#*:}"
+    iptables -t "$_tbl" -L "$_chain" -n -v --line-numbers 2>/dev/null \
+      | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v tbl="$_tbl" -v chain="$_chain" '
+        /^[0-9]+/ && $2 + 0 > 0 {
+          ln = $1; pkts = $2; bytes = $3; target = $4
+          printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=iptables tbl=%s chain=%s line=%s pkts=%s bytes=%s target=%s\n",
+            ts, role, host, ts, tbl, chain, ln, pkts, bytes, target
+        }
+      ' | head -n 8 >> "$WAN_WATCH_LOG" 2>/dev/null || true
+  done
+}
+
+wan_deep_qdisc() {
+  _parent_ts="$1"
+  _role="$2"
+  _hostn="$3"
+  _dev="$4"
+  [ -n "$_dev" ] || return 0
+  tc -s qdisc show dev "$_dev" 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v dev="$_dev" '
+    /qdisc/ {
+      printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=qdisc dev=%s detail=%s\n",
+        ts, role, host, ts, dev, $0
+    }
+    /backlog/ {
+      printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=qdisc dev=%s backlog=%s\n",
+        ts, role, host, ts, dev, $0
+    }
+  ' >> "$WAN_WATCH_LOG" 2>/dev/null || true
+}
+
+wan_deep_ss_summary() {
+  _parent_ts="$1"
+  _role="$2"
+  _hostn="$3"
+  ss -s 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" '
+    /TCP:|UDP:/ {
+      gsub(/^[ \t]+/, "")
+      printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=ss %s\n", ts, role, host, ts, $0
+    }
+  ' >> "$WAN_WATCH_LOG" 2>/dev/null || true
+}
+
+# Full deep capture for one anomaly event.
+wan_deep_diag_run() {
+  _parent_ts="$1"
+  _role="$2"
+  _hostn="$3"
+  _triggers="$4"
+  _wan_target="$5"
+  _fe_target="$6"
+  _gw_target="$7"
+  _route_wan_dev="$8"
+  _route_fe_dev="$9"
+
+  wan_deep_emit alert "$_parent_ts" "$_role" "$_hostn" \
+    "parent_ts=$_parent_ts triggers=\"$_triggers\""
+
+  wan_deep_host_health "$_parent_ts" "$_role" "$_hostn"
+  wan_deep_burst_ping "$_parent_ts" "$_role" "$_hostn" "wan" "$_wan_target"
+  wan_deep_burst_ping "$_parent_ts" "$_role" "$_hostn" "fe" "$_fe_target"
+  if [ -n "$_gw_target" ] && [ "$_gw_target" != "none" ]; then
+    wan_deep_burst_ping "$_parent_ts" "$_role" "$_hostn" "gw" "$_gw_target"
+  fi
+
+  wan_deep_route "$_parent_ts" "$_role" "$_hostn" "wan" "$_wan_target"
+  wan_deep_route "$_parent_ts" "$_role" "$_hostn" "fe" "$_fe_target"
+  wan_deep_neigh "$_parent_ts" "$_role" "$_hostn" "$_fe_target"
+  wan_deep_top_procs "$_parent_ts" "$_role" "$_hostn" 5
+  wan_deep_softnet "$_parent_ts" "$_role" "$_hostn"
+  wan_deep_ss_summary "$_parent_ts" "$_role" "$_hostn"
+
+  wan_deep_iface_stats "$_parent_ts" "$_role" "$_hostn" "$_route_wan_dev"
+  wan_deep_iface_stats "$_parent_ts" "$_role" "$_hostn" "$_route_fe_dev"
+  wan_deep_all_ifaces_nonzero "$_parent_ts" "$_role" "$_hostn"
+
+  if [ "$_role" = "router" ]; then
+    wan_deep_iptables_hot "$_parent_ts" "$_role" "$_hostn"
+    wan_deep_qdisc "$_parent_ts" "$_role" "$_hostn" "$_route_wan_dev"
+  fi
+
+  if [ "$_role" = "pi" ]; then
+    wan_deep_qdisc "$_parent_ts" "$_role" "$_hostn" "$_route_fe_dev"
+  fi
+}
+
+# Loaded marker for wan-snapshot.sh
+WAN_DEEP_DIAG_LOADED=1
--- a/examples/network/wan-snapshot.env.example
+++ b/examples/network/wan-snapshot.env.example
@@ -1,27 +1,36 @@
 # Copy to /etc/conf.d/wan-snapshot on each host (optional).
-# Source from cron:  . /etc/conf.d/wan-snapshot 2>/dev/null; /usr/local/bin/wan-snapshot.sh
+# Cron: . /etc/conf.d/wan-snapshot 2>/dev/null; /usr/local/bin/wan-snapshot.sh >>/var/log/wan-quality.cron.log 2>&1
+#
+# Deploy BOTH scripts:
+#   cp wan-snapshot.sh wan-deep-diag.sh /usr/local/bin/

-# --- Raspberry Pi (external / home vantage) ---
+# Unified log (snapshot + auto alert/deep on anomaly):
+WAN_WATCH_LOG=/var/log/wan-watch.log
+# Legacy alias — if same as WAN_WATCH_LOG, only one file is written:
+WAN_SNAPSHOT_LOG=/var/log/wan-watch.log
+
+WAN_SNAPSHOT_WATCH=1
+WAN_WATCH_COOLDOWN_S=840
+
+# --- Raspberry Pi ---
 # WAN_SNAPSHOT_ROLE=pi
-# WAN_SNAPSHOT_LOG=/var/log/wan-quality.log
 # ROUTER_PING_TARGET=10.7.6.228
 # FE_PING_TARGET=10.7.0.10
 # BE_PING_TARGET=10.7.16.128

+# --- Debian edge (monstro) ---
+# WAN_SNAPSHOT_ROLE=router
+# FE_PING_TARGET=10.7.0.10
+# BE_PING_TARGET=10.7.16.128
+
 # --- Gentoo FE ---
 # WAN_SNAPSHOT_ROLE=fe
-# WAN_SNAPSHOT_LOG=/var/log/wan-quality.log
 # FE_PING_TARGET=10.7.0.10
 # BE_PING_TARGET=10.7.16.128
 # ROUTER_PING_TARGET=10.7.6.228

-# --- Debian edge router (monstro) ---
-# WAN_SNAPSHOT_ROLE=router
-# WAN_SNAPSHOT_LOG=/var/log/wan-quality.log
-# FE_PING_TARGET=10.7.0.10
-# BE_PING_TARGET=10.7.16.128
-
-# Shared
-# WAN_SNAPSHOT_EXTRAS=1
-# PING_COUNT=10
-# PING_DEADLINE_S=15
+# Thresholds (ms / %)
+# WAN_ALERT_MS=50
+# FE_ALERT_MS=50
+# WAN_MDEV_ALERT_MS=80
+# WAN_CONNTRACK_ALERT_PCT=80
--- a/examples/network/wan-snapshot.sh
+++ b/examples/network/wan-snapshot.sh
@@ -15,24 +15,40 @@
 #   */15 * * * * root /usr/local/bin/wan-snapshot.sh >>/var/log/wan-quality.cron.log 2>&1
 #
 # Env overrides:
-#   WAN_SNAPSHOT_LOG=/var/log/wan-quality.log
-#   WAN_SNAPSHOT_ROLE=auto|router|fe|pi     (default auto from hostname)
-#   WAN_SNAPSHOT_EXTRAS=1                   (0 = ping only, compact v1-style line)
+#   WAN_WATCH_LOG=/var/log/wan-watch.log     (unified: snapshot + alert + deep)
+#   WAN_SNAPSHOT_LOG=/var/log/wan-quality.log (legacy snapshot copy if different)
+#   WAN_SNAPSHOT_ROLE=auto|router|fe|pi
+#   WAN_SNAPSHOT_EXTRAS=1
+#   WAN_SNAPSHOT_WATCH=1                    (0 = disable auto deep-diag on anomaly)
 #   WAN_SNAPSHOT_LOCK=/var/run/wan-snapshot.lock
-#   WAN_PING_TARGET=1.1.1.1
-#   FE_PING_TARGET=10.7.0.10
-#   BE_PING_TARGET=10.7.16.128              (set to "none" to skip)
-#   ROUTER_PING_TARGET=none|10.7.6.228      (optional; useful on Pi)
-#   GW_PING_TARGET=auto|none|<ip>           (default auto = default gateway)
-#   PING_COUNT=10
-#   PING_DEADLINE_S=15
+#   WAN_WATCH_COOLDOWN_S=840                  (min seconds between deep captures)
+#   WAN_ALERT_MS=50 FE_ALERT_MS=50 BE_ALERT_MS=50 GW_ALERT_MS=50
+#   WAN_LOSS_ALERT_PCT=1  WAN_MDEV_ALERT_MS=80  WAN_CONNTRACK_ALERT_PCT=80
+#   WAN_PING_TARGET=1.1.1.1  FE_PING_TARGET=10.7.0.10
+#   BE_PING_TARGET=10.7.16.128  ROUTER_PING_TARGET=none  GW_PING_TARGET=auto
+#   PING_COUNT=10  PING_DEADLINE_S=15
+#
+# Install deep-diag library next to this script:
+#   cp wan-snapshot.sh wan-deep-diag.sh /usr/local/bin/
 #
 set -eu

 WAN_SNAPSHOT_LOG="${WAN_SNAPSHOT_LOG:-/var/log/wan-quality.log}"
+WAN_WATCH_LOG="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
 WAN_SNAPSHOT_ROLE="${WAN_SNAPSHOT_ROLE:-auto}"
 WAN_SNAPSHOT_EXTRAS="${WAN_SNAPSHOT_EXTRAS:-1}"
+WAN_SNAPSHOT_WATCH="${WAN_SNAPSHOT_WATCH:-1}"
 WAN_SNAPSHOT_LOCK="${WAN_SNAPSHOT_LOCK:-/var/run/wan-snapshot.lock}"
+WAN_WATCH_COOLDOWN_S="${WAN_WATCH_COOLDOWN_S:-840}"
+WAN_WATCH_COOLDOWN_FILE="${WAN_WATCH_COOLDOWN_FILE:-/var/run/wan-watch-deep.stamp}"
+WAN_ALERT_MS="${WAN_ALERT_MS:-50}"
+FE_ALERT_MS="${FE_ALERT_MS:-50}"
+BE_ALERT_MS="${BE_ALERT_MS:-50}"
+GW_ALERT_MS="${GW_ALERT_MS:-50}"
+ROUTER_ALERT_MS="${ROUTER_ALERT_MS:-50}"
+WAN_LOSS_ALERT_PCT="${WAN_LOSS_ALERT_PCT:-1}"
+WAN_MDEV_ALERT_MS="${WAN_MDEV_ALERT_MS:-80}"
+WAN_CONNTRACK_ALERT_PCT="${WAN_CONNTRACK_ALERT_PCT:-80}"
 WAN_PING_TARGET="${WAN_PING_TARGET:-1.1.1.1}"
 FE_PING_TARGET="${FE_PING_TARGET:-10.7.0.10}"
 BE_PING_TARGET="${BE_PING_TARGET-10.7.16.128}"
@@ -41,6 +57,13 @@ GW_PING_TARGET="${GW_PING_TARGET:-auto}"
 PING_COUNT="${PING_COUNT:-10}"
 PING_DEADLINE_S="${PING_DEADLINE_S:-15}"

+_WAN_SNAPSHOT_DIR="$(CDPATH= cd -- "$(dirname "$0")" && pwd)"
+# shellcheck source=/dev/null
+. "$_WAN_SNAPSHOT_DIR/wan-deep-diag.sh" 2>/dev/null \
+  || . /usr/local/bin/wan-deep-diag.sh 2>/dev/null \
+  || . /usr/local/lib/wan-deep-diag.sh 2>/dev/null \
+  || true
+
 usage() {
  sed -n '2,28p' "$0"
  printf '\nCommands:\n'
@@ -48,6 +71,7 @@ usage() {
  printf '  --report            summarize today vs yesterday\n'
  printf '  --tail [N]          print last N log lines (default 24)\n'
  printf '  --correlate FILE…   merge logs by 15-min slot (role= or path basename)\n'
+  printf '  --alerts [N]        recent kind=alert + kind=deep lines (default 40)\n'
  printf '  --help              this text\n'
 }

@@ -182,12 +206,149 @@ iface_drops_compact() {
  ' | sed 's/ $//'
 }

+# True if line is a snapshot row (legacy ts=… or kind=snapshot).
+is_snapshot_line() {
+  case "$1" in
+    kind=snapshot*) return 0 ;;
+    ts=*) return 0 ;;
+    *) return 1 ;;
+  esac
+}
+
+num_ge() {
+  _a="$1"
+  _b="$2"
+  [ "$_a" != "-1" ] && [ -n "$_a" ] && awk -v a="$_a" -v b="$_b" 'BEGIN { exit (a + 0 >= b + 0) ? 0 : 1 }'
+}
+
+watch_cooldown_active() {
+  [ -r "$WAN_WATCH_COOLDOWN_FILE" ] || return 1
+  _last="$(cat "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null)" || return 1
+  _now="$(date +%s 2>/dev/null)" || return 1
+  [ "$((_now - _last))" -lt "$WAN_WATCH_COOLDOWN_S" ] 2>/dev/null
+}
+
+watch_cooldown_stamp() {
+  date +%s > "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null || true
+}
+
+# Returns trigger list (comma-separated) or empty.
+detect_anomalies() {
+  _line="$1"
+  _role="$2"
+  _triggers=""
+
+  for _pfx in wan fe be gw router; do
+    _avg="$(field_value "$_line" "${_pfx}_avg_ms")"
+    _loss="$(field_value "$_line" "${_pfx}_loss")"
+    _mdev="$(field_value "$_line" "${_pfx}_mdev_ms")"
+    _max="$(field_value "$_line" "${_pfx}_max_ms")"
+    _thr="$FE_ALERT_MS"
+    case "$_pfx" in
+      wan) _thr="$WAN_ALERT_MS" ;;
+      fe) _thr="$FE_ALERT_MS" ;;
+      be) _thr="$BE_ALERT_MS" ;;
+      gw) _thr="$GW_ALERT_MS" ;;
+      router) _thr="$ROUTER_ALERT_MS" ;;
+    esac
+    if num_ge "$_avg" "$_thr"; then
+      _triggers="${_triggers}${_triggers:+,}${_pfx}_avg=${_avg}"
+    fi
+    if num_ge "$_loss" "$WAN_LOSS_ALERT_PCT"; then
+      _triggers="${_triggers}${_triggers:+,}${_pfx}_loss=${_loss}"
+    fi
+    if num_ge "$_mdev" "$WAN_MDEV_ALERT_MS"; then
+      _triggers="${_triggers}${_triggers:+,}${_pfx}_mdev=${_mdev}"
+    fi
+    if num_ge "$_max" "$((_thr * 3))"; then
+      _triggers="${_triggers}${_triggers:+,}${_pfx}_max=${_max}"
+    fi
+  done
+
+  _neigh="$(printf '%s\n' "$_line" | sed -n 's/.*neigh_fe=\([^ ]*\).*/\1/p')"
+  if [ -n "$_neigh" ] && [ "$_neigh" != "REACHABLE" ] && [ "$_neigh" != "DELAY" ]; then
+    _triggers="${_triggers}${_triggers:+,}neigh_fe=${_neigh}"
+  fi
+
+  _ct_pct="$(printf '%s\n' "$_line" | sed -n 's/.*(\([0-9]*\)%).*/\1/p' | head -1)"
+  if [ -n "$_ct_pct" ] && num_ge "$_ct_pct" "$WAN_CONNTRACK_ALERT_PCT"; then
+    _triggers="${_triggers}${_triggers:+,}conntrack_pct=${_ct_pct}"
+  fi
+
+  # Pi: WAN ok but FE slow -> path anomaly (even if FE just above threshold).
+  if [ "$_role" = "pi" ]; then
+    _wan_avg="$(field_value "$_line" "wan_avg_ms")"
+    _fe_avg="$(field_value "$_line" "fe_avg_ms")"
+    if num_ge "$_fe_avg" "$FE_ALERT_MS" && ! num_ge "$_wan_avg" "$WAN_ALERT_MS"; then
+      case "$_triggers" in *fe_avg=*) ;; *)
+        _triggers="${_triggers}${_triggers:+,}pi_fe_path" ;;
+      esac
+    fi
+  fi
+
+  # Router: WAN bad, FE ok -> edge WAN leg (matches monstro xenbr0 pattern).
+  if [ "$_role" = "router" ]; then
+    _wan_avg="$(field_value "$_line" "wan_avg_ms")"
+    _fe_avg="$(field_value "$_line" "fe_avg_ms")"
+    if num_ge "$_wan_avg" "$WAN_ALERT_MS" && ! num_ge "$_fe_avg" "$FE_ALERT_MS"; then
+      case "$_triggers" in *wan_avg=*) ;; *)
+        _triggers="${_triggers}${_triggers:+,}router_wan_path" ;;
+      esac
+    fi
+  fi
+
+  printf '%s' "$_triggers"
+}
+
+maybe_deep_diag() {
+  _line="$1"
+  _ts="$2"
+  _role="$3"
+  _hostn="$4"
+
+  [ "$WAN_SNAPSHOT_WATCH" = "1" ] || return 0
+  [ "${WAN_DEEP_DIAG_LOADED:-0}" = "1" ] || {
+    echo "wan-snapshot: watch enabled but wan-deep-diag.sh not loaded (install alongside wan-snapshot.sh)" >&2
+    return 0
+  }
+
+  _triggers="$(detect_anomalies "$_line" "$_role")"
+  [ -n "$_triggers" ] || return 0
+
+  if watch_cooldown_active; then
+    wan_deep_emit alert "$_ts" "$_role" "$_hostn" \
+      "parent_ts=$_ts triggers=\"$_triggers\" action=skipped_cooldown"
+    return 0
+  fi
+
+  _gw_target="$GW_PING_TARGET"
+  if [ "$_gw_target" = "auto" ]; then
+    _gw_target="$(default_gw)"
+  fi
+  _wdev="$(route_dev "$WAN_PING_TARGET")"
+  _fdev="$(route_dev "$FE_PING_TARGET")"
+
+  wan_deep_diag_run "$_ts" "$_role" "$_hostn" "$_triggers" \
+    "$WAN_PING_TARGET" "$FE_PING_TARGET" "$_gw_target" "$_wdev" "$_fdev"
+  watch_cooldown_stamp
+}
+
+append_watch_log() {
+  _line="$1"
+  _dir="$(dirname "$WAN_WATCH_LOG")"
+  [ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true
+  printf '%s\n' "$_line" >> "$WAN_WATCH_LOG"
+}
+
 acquire_lock() {
  command -v flock >/dev/null 2>&1 || return 0
  _lock="$WAN_SNAPSHOT_LOCK"
  _dir="$(dirname "$_lock")"
-  if ! mkdir -p "$_dir" 2>/dev/null || ! : >"$_lock" 2>/dev/null; then
-    _lock="${TMPDIR:-/tmp}/wan-snapshot.$(id -u 2>/dev/null || echo 0).lock"
+  _fallback="${TMPDIR:-/tmp}/wan-snapshot.$(id -u 2>/dev/null || echo 0).lock"
+  if [ ! -d "$_dir" ] || [ ! -w "$_dir" ] 2>/dev/null; then
+    _lock="$_fallback"
+  elif ! ( : >>"$_lock" ) 2>/dev/null; then
+    _lock="$_fallback"
  fi
  exec 9>"$_lock" 2>/dev/null || return 0
  flock -n 9 || {
@@ -204,7 +365,7 @@ snapshot_once() {
  _role="$(detect_role)"
  _host="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)"

-  _line="ts=$_ts ver=2 role=$_role host=$_host"
+  _line="kind=snapshot ts=$_ts ver=2 role=$_role host=$_host"
  _line="$(append_ping_fields "$_line" "wan" "$WAN_PING_TARGET")"
  _line="$(append_ping_fields "$_line" "fe" "$FE_PING_TARGET")"

@@ -241,9 +402,15 @@ snapshot_once() {
    [ -n "$_drops" ] && _line="$_line drops=\"$_drops\""
  fi

-  _dir="$(dirname "$WAN_SNAPSHOT_LOG")"
-  [ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true
-  printf '%s\n' "$_line" >> "$WAN_SNAPSHOT_LOG"
+  append_watch_log "$_line"
+  if [ "$WAN_SNAPSHOT_LOG" != "$WAN_WATCH_LOG" ]; then
+    _dir="$(dirname "$WAN_SNAPSHOT_LOG")"
+    [ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true
+    printf '%s\n' "$_line" >> "$WAN_SNAPSHOT_LOG"
+  fi
+
+  maybe_deep_diag "$_line" "$_ts" "$_role" "$_host"
+
  printf '%s\n' "$_line"
 }

@@ -271,7 +438,7 @@ day_stats() {
  _field="$3"
  [ -r "$_log" ] || return 0
  awk -v d="$_date" -v f="$_field" '
-    $0 ~ "^ts=" d {
+    ($0 ~ "^kind=snapshot ts=" d || ($0 ~ "^ts=" d && $0 !~ / kind=/)) {
      for (i = 1; i <= NF; i++) {
        if ($i ~ "^" f "=") {
          split($i, a, "=")
@@ -302,7 +469,7 @@ day_max_loss() {
  _field="$3"
  [ -r "$_log" ] || return 0
  awk -v d="$_date" -v f="$_field" '
-    $0 ~ "^ts=" d {
+    ($0 ~ "^kind=snapshot ts=" d || ($0 ~ "^ts=" d && $0 !~ / kind=/)) {
      for (i = 1; i <= NF; i++) {
        if ($i ~ "^" f "=") {
          split($i, a, "=")
@@ -323,7 +490,7 @@ report_day_block() {
  _log="$1"
  _label="$2"
  _date="$3"
-  _samples="$(grep -c "^ts=$_date" "$_log" 2>/dev/null || echo 0)"
+  _samples="$(grep -cE "^kind=snapshot ts=$_date|^ts=$_date" "$_log" 2>/dev/null || echo 0)"
  printf '%s=%s samples=%s\n' "$_label" "$_date" "$_samples"
  for _pfx in wan fe be router gw; do
    _stats="$(day_stats "$_log" "$_date" "${_pfx}_avg_ms")"
@@ -344,7 +511,7 @@ report_day_block() {
 }

 report() {
-  _log="$WAN_SNAPSHOT_LOG"
+  _log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
  _today="$(date '+%Y-%m-%d')"
  _yesterday="$(date -d 'yesterday' '+%Y-%m-%d' 2>/dev/null || date -v-1d '+%Y-%m-%d' 2>/dev/null || echo '')"

@@ -357,7 +524,7 @@ report() {
  printf 'log=%s role=%s\n' "$_log" "$_role"
  report_day_block "$_log" "today" "$_today"

-  if [ -n "$_yesterday" ] && grep -q "^ts=$_yesterday" "$_log" 2>/dev/null; then
+  if [ -n "$_yesterday" ] && grep -qE "^kind=snapshot ts=$_yesterday|^ts=$_yesterday" "$_log" 2>/dev/null; then
    report_day_block "$_log" "yesterday" "$_yesterday"
  fi

@@ -373,11 +540,22 @@ report() {

 tail_log() {
  _n="${1:-24}"
-  [ -r "$WAN_SNAPSHOT_LOG" ] || {
-    echo "wan-snapshot: no log at $WAN_SNAPSHOT_LOG" >&2
+  _log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
+  [ -r "$_log" ] || {
+    echo "wan-snapshot: no log at $_log" >&2
    exit 1
  }
-  tail -n "$_n" "$WAN_SNAPSHOT_LOG"
+  grep -E '^kind=snapshot |^ts=' "$_log" 2>/dev/null | tail -n "$_n"
+}
+
+tail_alerts() {
+  _n="${1:-40}"
+  _log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
+  [ -r "$_log" ] || {
+    echo "wan-snapshot: no log at $_log" >&2
+    exit 1
+  }
+  grep -E '^kind=alert |^kind=deep ' "$_log" 2>/dev/null | tail -n "$_n"
 }

 # Guess role label for correlate when log line has no role=.
@@ -428,11 +606,12 @@ correlate_logs() {
    }
    _label="$(basename "$_path")"
    while IFS= read -r _line || [ -n "$_line" ]; do
+      is_snapshot_line "$_line" || continue
      case "$_line" in
-        ts=*) ;;
+        kind=snapshot*) _ts="${_line#kind=snapshot ts=}" ;;
+        ts=*) _ts="${_line#ts=}" ;;
        *) continue ;;
      esac
-      _ts="${_line#ts=}"
      _ts="${_ts%% *}"
      _slot="$(slot_key "$_ts")"
      _role="$(log_role_label "$_path" "$_line")"
@@ -500,6 +679,9 @@ case "${1:-}" in
  --tail|tail)
    tail_log "${2:-24}"
    ;;
+  --alerts|alerts)
+    tail_alerts "${2:-40}"
+    ;;
  --correlate|correlate)
    shift
    correlate_logs "$@"