diff --git a/examples/network/README.md b/examples/network/README.md index d24fc0d..285c855 100644 --- a/examples/network/README.md +++ b/examples/network/README.md @@ -2,6 +2,52 @@ Scripts for **router**, **FE**, and **external vantage** (e.g. Raspberry Pi) — not BE app hosts. +## Auto watchdog (v2.1) + +When a cron snapshot exceeds thresholds, `wan-snapshot.sh` appends **alert** + **deep** lines to the same log (`WAN_WATCH_LOG`). + +| `kind=` | Meaning | +|---------|---------| +| `snapshot` | Normal 15-min ping row | +| `alert` | Threshold breached; lists `triggers=` | +| `deep` | One parseable fact (`section=host`, `burst`, `iface`, `iptables`, …) | + +**Deploy both files:** + +```bash +sudo cp examples/network/wan-snapshot.sh examples/network/wan-deep-diag.sh /usr/local/bin/ +sudo chmod +x /usr/local/bin/wan-snapshot.sh /usr/local/bin/wan-deep-diag.sh +``` + +**Env** (`/etc/conf.d/wan-snapshot`): + +```sh +WAN_WATCH_LOG=/var/log/wan-watch.log +WAN_SNAPSHOT_LOG=/var/log/wan-watch.log +WAN_SNAPSHOT_WATCH=1 +``` + +**Triggers (defaults):** `*_avg_ms >= 50`, `*_loss >= 1%`, `mdev >= 80`, `conntrack >= 80%`, `neigh_fe` not REACHABLE/DELAY, plus role hints `pi_fe_path` / `router_wan_path`. + +**Cooldown:** one deep capture per 14 min (`WAN_WATCH_COOLDOWN_S=840`) to avoid cron pile-up. + +**Review after a bad spell:** + +```bash +/usr/local/bin/wan-snapshot.sh --alerts 80 +/usr/local/bin/wan-snapshot.sh --correlate /var/log/wan-watch.log … +grep 'kind=deep.*section=burst' /var/log/wan-watch.log | tail +grep 'kind=deep.*section=iptables' /var/log/wan-watch.log | tail +``` + +Example deep lines: + +```text +kind=alert ts=2026-06-12T21:45:01+03:00 role=router host=monstro parent_ts=... triggers="wan_avg=535.365,router_wan_path" +kind=deep ts=2026-06-12T21:45:01+03:00 role=router host=monstro parent_ts=... section=burst label=wan target=1.1.1.1 loss=0% avg_ms=520 max_ms=790 mdev_ms=110 +kind=deep ts=2026-06-12T21:45:01+03:00 role=router host=monstro parent_ts=... section=iptables tbl=filter chain=FORWARD line=3 pkts=188000 target=ACCEPT +``` + ## `wan-snapshot.sh` (v2) Periodic ping comparison plus lightweight host/path context. diff --git a/examples/network/wan-deep-diag.sh b/examples/network/wan-deep-diag.sh new file mode 100644 index 0000000..10ac8e4 --- /dev/null +++ b/examples/network/wan-deep-diag.sh @@ -0,0 +1,287 @@ +# wan-deep-diag.sh — parseable deep diagnostics (sourced by wan-snapshot.sh). +# Not meant to be executed directly. +# +# Emits kind=deep lines to WAN_WATCH_LOG (key=value, one fact per line). + +wan_deep_load_1m() { + awk '{print $1}' /proc/loadavg 2>/dev/null || echo "n/a" +} + +wan_deep_cpu_busy_pct() { + awk '/^cpu / { + idle = $5 + $6; total = 0 + for (i = 2; i <= NF; i++) total += $i + if (total > 0) printf "%.0f", 100 * (total - idle) / total + }' /proc/stat 2>/dev/null || echo "n/a" +} + +wan_deep_mem_avail_kb() { + awk '/^MemAvailable:/ { print $2 }' /proc/meminfo 2>/dev/null || echo "n/a" +} + +wan_deep_conntrack_fields() { + _cur="$(cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null)" || _cur="" + _max="$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null)" || _max="" + if [ -n "$_cur" ] && [ -n "$_max" ] && [ "$_max" -gt 0 ] 2>/dev/null; then + _pct=$((_cur * 100 / _max)) + printf 'conntrack_cur=%s conntrack_max=%s conntrack_pct=%s' "$_cur" "$_max" "$_pct" + else + printf 'conntrack_cur=n/a conntrack_max=n/a conntrack_pct=n/a' + fi +} + +wan_deep_emit() { + # wan_deep_emit KIND TS ROLE HOST [key=value ...] + _kind="$1" + _ts="$2" + _role="$3" + _host="$4" + shift 4 + _line="kind=$_kind ts=$_ts role=$_role host=$_host $*" + _dir="$(dirname "$WAN_WATCH_LOG")" + [ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true + printf '%s\n' "$_line" >> "$WAN_WATCH_LOG" +} + +# Burst ping (fast) -> one kind=deep line. +wan_deep_burst_ping() { + _parent_ts="$1" + _role="$2" + _hostn="$3" + _label="$4" + _target="$5" + _count="${6:-5}" + _deadline="${7:-6}" + + _out="$(ping -c "$_count" -w "$_deadline" "$_target" 2>/dev/null)" || _out="" + if [ -z "$_out" ]; then + wan_deep_emit deep "$_parent_ts" "$_role" "$_hostn" \ + "parent_ts=$_parent_ts section=burst label=$_label target=$_target loss=-1 avg_ms=-1" + return 0 + fi + + _loss="$(printf '%s\n' "$_out" | awk '/packet loss/ { + for (i = 1; i <= NF; i++) if ($i ~ /%$/) { gsub(/%/, "", $i); print $i; exit } + }')" + _rtt="$(printf '%s\n' "$_out" | sed -n 's/.*= \([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\) ms.*/\1 \2 \3 \4/p' | head -1)" + _min="-1" _avg="-1" _max="-1" _mdev="-1" + if [ -n "$_rtt" ]; then + _min="${_rtt%% *}" + _rest="${_rtt#* }" + _avg="${_rest%% *}" + _rest="${_rest#* }" + _max="${_rest%% *}" + _mdev="${_rest#* }" + fi + [ -n "$_loss" ] || _loss="-1" + + wan_deep_emit deep "$_parent_ts" "$_role" "$_hostn" \ + "parent_ts=$_parent_ts section=burst label=$_label target=$_target loss=${_loss}% min_ms=$_min avg_ms=$_avg max_ms=$_max mdev_ms=$_mdev" +} + +wan_deep_host_health() { + _parent_ts="$1" + _role="$2" + _hostn="$3" + _ct="$(wan_deep_conntrack_fields)" + wan_deep_emit deep "$_parent_ts" "$_role" "$_hostn" \ + "parent_ts=$_parent_ts section=host load1=$(wan_deep_load_1m) cpu_busy=$(wan_deep_cpu_busy_pct)% mem_avail_kb=$(wan_deep_mem_avail_kb) $_ct" +} + +wan_deep_top_procs() { + _parent_ts="$1" + _role="$2" + _hostn="$3" + _n="${4:-5}" + ps -eo pcpu=,pmem=,comm= --sort=-pcpu 2>/dev/null | head -n "$_n" | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v n="$_n" ' + NF >= 3 { + rank = NR + printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=proc rank=%d cpu=%s mem=%s comm=%s\n", + ts, role, host, ts, rank, $1, $2, $3 + } + ' >> "$WAN_WATCH_LOG" 2>/dev/null || true +} + +wan_deep_iface_stats() { + _parent_ts="$1" + _role="$2" + _hostn="$3" + _dev="$4" + [ -n "$_dev" ] || return 0 + ip -s link show "$_dev" 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v dev="$_dev" ' + /^[0-9]+:/ { rx_b = tx_b = rx_p = tx_p = rx_drop = tx_drop = rx_err = tx_err = 0 } + /RX: bytes/ { rx_b = $3; rx_p = $2 } + /RX:.*packets/ { getline; rx_err = $3; rx_drop = $4 } + /TX: bytes/ { tx_b = $3; tx_p = $2 } + /TX:.*packets/ { getline; tx_err = $3; tx_drop = $4 } + END { + printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=iface dev=%s rx_bytes=%s tx_bytes=%s rx_drop=%s tx_drop=%s rx_err=%s tx_err=%s\n", + ts, role, host, ts, dev, rx_b, tx_b, rx_drop, tx_drop, rx_err, tx_err + } + ' >> "$WAN_WATCH_LOG" 2>/dev/null || true +} + +wan_deep_all_ifaces_nonzero() { + _parent_ts="$1" + _role="$2" + _hostn="$3" + ip -s link 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" ' + /^[0-9]+:/ { + if (iface != "" && iface !~ /^lo$/ && (rx_drop + tx_drop + rx_err + tx_err) > 0) + printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=iface dev=%s rx_drop=%s tx_drop=%s rx_err=%s tx_err=%s\n", + ts, role, host, ts, iface, rx_drop, tx_drop, rx_err, tx_err + iface = $2; gsub(":", "", iface) + rx_drop = tx_drop = rx_err = tx_err = 0 + } + /RX:/ { getline; rx_err = $3; rx_drop = $4 } + /TX:/ { getline; tx_err = $3; tx_drop = $4 } + END { + if (iface != "" && iface !~ /^lo$/ && (rx_drop + tx_drop + rx_err + tx_err) > 0) + printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=iface dev=%s rx_drop=%s tx_drop=%s rx_err=%s tx_err=%s\n", + ts, role, host, ts, iface, rx_drop, tx_drop, rx_err, tx_err + } + ' >> "$WAN_WATCH_LOG" 2>/dev/null || true +} + +wan_deep_softnet() { + _parent_ts="$1" + _role="$2" + _hostn="$3" + [ -r /proc/net/softnet_stat ] || return 0 + awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" ' + $2 + 0 > 0 { + printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=softnet cpu=%d dropped=%s processed=%s\n", + ts, role, host, ts, NR - 1, $2, $1 + } + ' /proc/net/softnet_stat >> "$WAN_WATCH_LOG" 2>/dev/null || true +} + +wan_deep_neigh() { + _parent_ts="$1" + _role="$2" + _hostn="$3" + _ip="$4" + [ -n "$_ip" ] || return 0 + ip neigh show "$_ip" 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v ip="$_ip" ' + NF >= 5 { + printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=neigh ip=%s lladdr=%s dev=%s state=%s\n", + ts, role, host, ts, ip, $5, $3, $6 + } + ' >> "$WAN_WATCH_LOG" 2>/dev/null || true +} + +wan_deep_route() { + _parent_ts="$1" + _role="$2" + _hostn="$3" + _label="$4" + _target="$5" + [ -n "$_target" ] || return 0 + ip route get "$_target" 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v label="$_label" -v target="$_target" ' + { + dev = via = ""; src = "" + for (i = 1; i <= NF; i++) { + if ($i == "dev") dev = $(i + 1) + if ($i == "via") via = $(i + 1) + if ($i == "src") src = $(i + 1) + } + printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=route label=%s target=%s dev=%s via=%s src=%s\n", + ts, role, host, ts, label, target, dev, via, src + } + ' >> "$WAN_WATCH_LOG" 2>/dev/null || true +} + +wan_deep_iptables_hot() { + _parent_ts="$1" + _role="$2" + _hostn="$3" + command -v iptables >/dev/null 2>&1 || return 0 + for _spec in "filter:FORWARD" "nat:PREROUTING" "nat:POSTROUTING"; do + _tbl="${_spec%%:*}" + _chain="${_spec#*:}" + iptables -t "$_tbl" -L "$_chain" -n -v --line-numbers 2>/dev/null \ + | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v tbl="$_tbl" -v chain="$_chain" ' + /^[0-9]+/ && $2 + 0 > 0 { + ln = $1; pkts = $2; bytes = $3; target = $4 + printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=iptables tbl=%s chain=%s line=%s pkts=%s bytes=%s target=%s\n", + ts, role, host, ts, tbl, chain, ln, pkts, bytes, target + } + ' | head -n 8 >> "$WAN_WATCH_LOG" 2>/dev/null || true + done +} + +wan_deep_qdisc() { + _parent_ts="$1" + _role="$2" + _hostn="$3" + _dev="$4" + [ -n "$_dev" ] || return 0 + tc -s qdisc show dev "$_dev" 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v dev="$_dev" ' + /qdisc/ { + printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=qdisc dev=%s detail=%s\n", + ts, role, host, ts, dev, $0 + } + /backlog/ { + printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=qdisc dev=%s backlog=%s\n", + ts, role, host, ts, dev, $0 + } + ' >> "$WAN_WATCH_LOG" 2>/dev/null || true +} + +wan_deep_ss_summary() { + _parent_ts="$1" + _role="$2" + _hostn="$3" + ss -s 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" ' + /TCP:|UDP:/ { + gsub(/^[ \t]+/, "") + printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=ss %s\n", ts, role, host, ts, $0 + } + ' >> "$WAN_WATCH_LOG" 2>/dev/null || true +} + +# Full deep capture for one anomaly event. +wan_deep_diag_run() { + _parent_ts="$1" + _role="$2" + _hostn="$3" + _triggers="$4" + _wan_target="$5" + _fe_target="$6" + _gw_target="$7" + _route_wan_dev="$8" + _route_fe_dev="$9" + + wan_deep_emit alert "$_parent_ts" "$_role" "$_hostn" \ + "parent_ts=$_parent_ts triggers=\"$_triggers\"" + + wan_deep_host_health "$_parent_ts" "$_role" "$_hostn" + wan_deep_burst_ping "$_parent_ts" "$_role" "$_hostn" "wan" "$_wan_target" + wan_deep_burst_ping "$_parent_ts" "$_role" "$_hostn" "fe" "$_fe_target" + if [ -n "$_gw_target" ] && [ "$_gw_target" != "none" ]; then + wan_deep_burst_ping "$_parent_ts" "$_role" "$_hostn" "gw" "$_gw_target" + fi + + wan_deep_route "$_parent_ts" "$_role" "$_hostn" "wan" "$_wan_target" + wan_deep_route "$_parent_ts" "$_role" "$_hostn" "fe" "$_fe_target" + wan_deep_neigh "$_parent_ts" "$_role" "$_hostn" "$_fe_target" + wan_deep_top_procs "$_parent_ts" "$_role" "$_hostn" 5 + wan_deep_softnet "$_parent_ts" "$_role" "$_hostn" + wan_deep_ss_summary "$_parent_ts" "$_role" "$_hostn" + + wan_deep_iface_stats "$_parent_ts" "$_role" "$_hostn" "$_route_wan_dev" + wan_deep_iface_stats "$_parent_ts" "$_role" "$_hostn" "$_route_fe_dev" + wan_deep_all_ifaces_nonzero "$_parent_ts" "$_role" "$_hostn" + + if [ "$_role" = "router" ]; then + wan_deep_iptables_hot "$_parent_ts" "$_role" "$_hostn" + wan_deep_qdisc "$_parent_ts" "$_role" "$_hostn" "$_route_wan_dev" + fi + + if [ "$_role" = "pi" ]; then + wan_deep_qdisc "$_parent_ts" "$_role" "$_hostn" "$_route_fe_dev" + fi +} + +# Loaded marker for wan-snapshot.sh +WAN_DEEP_DIAG_LOADED=1 \ No newline at end of file diff --git a/examples/network/wan-snapshot.env.example b/examples/network/wan-snapshot.env.example index 0cda965..4402a7c 100644 --- a/examples/network/wan-snapshot.env.example +++ b/examples/network/wan-snapshot.env.example @@ -1,27 +1,36 @@ # Copy to /etc/conf.d/wan-snapshot on each host (optional). -# Source from cron: . /etc/conf.d/wan-snapshot 2>/dev/null; /usr/local/bin/wan-snapshot.sh +# Cron: . /etc/conf.d/wan-snapshot 2>/dev/null; /usr/local/bin/wan-snapshot.sh >>/var/log/wan-quality.cron.log 2>&1 +# +# Deploy BOTH scripts: +# cp wan-snapshot.sh wan-deep-diag.sh /usr/local/bin/ -# --- Raspberry Pi (external / home vantage) --- +# Unified log (snapshot + auto alert/deep on anomaly): +WAN_WATCH_LOG=/var/log/wan-watch.log +# Legacy alias — if same as WAN_WATCH_LOG, only one file is written: +WAN_SNAPSHOT_LOG=/var/log/wan-watch.log + +WAN_SNAPSHOT_WATCH=1 +WAN_WATCH_COOLDOWN_S=840 + +# --- Raspberry Pi --- # WAN_SNAPSHOT_ROLE=pi -# WAN_SNAPSHOT_LOG=/var/log/wan-quality.log # ROUTER_PING_TARGET=10.7.6.228 # FE_PING_TARGET=10.7.0.10 # BE_PING_TARGET=10.7.16.128 +# --- Debian edge (monstro) --- +# WAN_SNAPSHOT_ROLE=router +# FE_PING_TARGET=10.7.0.10 +# BE_PING_TARGET=10.7.16.128 + # --- Gentoo FE --- # WAN_SNAPSHOT_ROLE=fe -# WAN_SNAPSHOT_LOG=/var/log/wan-quality.log # FE_PING_TARGET=10.7.0.10 # BE_PING_TARGET=10.7.16.128 # ROUTER_PING_TARGET=10.7.6.228 -# --- Debian edge router (monstro) --- -# WAN_SNAPSHOT_ROLE=router -# WAN_SNAPSHOT_LOG=/var/log/wan-quality.log -# FE_PING_TARGET=10.7.0.10 -# BE_PING_TARGET=10.7.16.128 - -# Shared -# WAN_SNAPSHOT_EXTRAS=1 -# PING_COUNT=10 -# PING_DEADLINE_S=15 +# Thresholds (ms / %) +# WAN_ALERT_MS=50 +# FE_ALERT_MS=50 +# WAN_MDEV_ALERT_MS=80 +# WAN_CONNTRACK_ALERT_PCT=80 diff --git a/examples/network/wan-snapshot.sh b/examples/network/wan-snapshot.sh index e55f566..c888d03 100755 --- a/examples/network/wan-snapshot.sh +++ b/examples/network/wan-snapshot.sh @@ -15,24 +15,40 @@ # */15 * * * * root /usr/local/bin/wan-snapshot.sh >>/var/log/wan-quality.cron.log 2>&1 # # Env overrides: -# WAN_SNAPSHOT_LOG=/var/log/wan-quality.log -# WAN_SNAPSHOT_ROLE=auto|router|fe|pi (default auto from hostname) -# WAN_SNAPSHOT_EXTRAS=1 (0 = ping only, compact v1-style line) +# WAN_WATCH_LOG=/var/log/wan-watch.log (unified: snapshot + alert + deep) +# WAN_SNAPSHOT_LOG=/var/log/wan-quality.log (legacy snapshot copy if different) +# WAN_SNAPSHOT_ROLE=auto|router|fe|pi +# WAN_SNAPSHOT_EXTRAS=1 +# WAN_SNAPSHOT_WATCH=1 (0 = disable auto deep-diag on anomaly) # WAN_SNAPSHOT_LOCK=/var/run/wan-snapshot.lock -# WAN_PING_TARGET=1.1.1.1 -# FE_PING_TARGET=10.7.0.10 -# BE_PING_TARGET=10.7.16.128 (set to "none" to skip) -# ROUTER_PING_TARGET=none|10.7.6.228 (optional; useful on Pi) -# GW_PING_TARGET=auto|none| (default auto = default gateway) -# PING_COUNT=10 -# PING_DEADLINE_S=15 +# WAN_WATCH_COOLDOWN_S=840 (min seconds between deep captures) +# WAN_ALERT_MS=50 FE_ALERT_MS=50 BE_ALERT_MS=50 GW_ALERT_MS=50 +# WAN_LOSS_ALERT_PCT=1 WAN_MDEV_ALERT_MS=80 WAN_CONNTRACK_ALERT_PCT=80 +# WAN_PING_TARGET=1.1.1.1 FE_PING_TARGET=10.7.0.10 +# BE_PING_TARGET=10.7.16.128 ROUTER_PING_TARGET=none GW_PING_TARGET=auto +# PING_COUNT=10 PING_DEADLINE_S=15 +# +# Install deep-diag library next to this script: +# cp wan-snapshot.sh wan-deep-diag.sh /usr/local/bin/ # set -eu WAN_SNAPSHOT_LOG="${WAN_SNAPSHOT_LOG:-/var/log/wan-quality.log}" +WAN_WATCH_LOG="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}" WAN_SNAPSHOT_ROLE="${WAN_SNAPSHOT_ROLE:-auto}" WAN_SNAPSHOT_EXTRAS="${WAN_SNAPSHOT_EXTRAS:-1}" +WAN_SNAPSHOT_WATCH="${WAN_SNAPSHOT_WATCH:-1}" WAN_SNAPSHOT_LOCK="${WAN_SNAPSHOT_LOCK:-/var/run/wan-snapshot.lock}" +WAN_WATCH_COOLDOWN_S="${WAN_WATCH_COOLDOWN_S:-840}" +WAN_WATCH_COOLDOWN_FILE="${WAN_WATCH_COOLDOWN_FILE:-/var/run/wan-watch-deep.stamp}" +WAN_ALERT_MS="${WAN_ALERT_MS:-50}" +FE_ALERT_MS="${FE_ALERT_MS:-50}" +BE_ALERT_MS="${BE_ALERT_MS:-50}" +GW_ALERT_MS="${GW_ALERT_MS:-50}" +ROUTER_ALERT_MS="${ROUTER_ALERT_MS:-50}" +WAN_LOSS_ALERT_PCT="${WAN_LOSS_ALERT_PCT:-1}" +WAN_MDEV_ALERT_MS="${WAN_MDEV_ALERT_MS:-80}" +WAN_CONNTRACK_ALERT_PCT="${WAN_CONNTRACK_ALERT_PCT:-80}" WAN_PING_TARGET="${WAN_PING_TARGET:-1.1.1.1}" FE_PING_TARGET="${FE_PING_TARGET:-10.7.0.10}" BE_PING_TARGET="${BE_PING_TARGET-10.7.16.128}" @@ -41,6 +57,13 @@ GW_PING_TARGET="${GW_PING_TARGET:-auto}" PING_COUNT="${PING_COUNT:-10}" PING_DEADLINE_S="${PING_DEADLINE_S:-15}" +_WAN_SNAPSHOT_DIR="$(CDPATH= cd -- "$(dirname "$0")" && pwd)" +# shellcheck source=/dev/null +. "$_WAN_SNAPSHOT_DIR/wan-deep-diag.sh" 2>/dev/null \ + || . /usr/local/bin/wan-deep-diag.sh 2>/dev/null \ + || . /usr/local/lib/wan-deep-diag.sh 2>/dev/null \ + || true + usage() { sed -n '2,28p' "$0" printf '\nCommands:\n' @@ -48,6 +71,7 @@ usage() { printf ' --report summarize today vs yesterday\n' printf ' --tail [N] print last N log lines (default 24)\n' printf ' --correlate FILE… merge logs by 15-min slot (role= or path basename)\n' + printf ' --alerts [N] recent kind=alert + kind=deep lines (default 40)\n' printf ' --help this text\n' } @@ -182,12 +206,149 @@ iface_drops_compact() { ' | sed 's/ $//' } +# True if line is a snapshot row (legacy ts=… or kind=snapshot). +is_snapshot_line() { + case "$1" in + kind=snapshot*) return 0 ;; + ts=*) return 0 ;; + *) return 1 ;; + esac +} + +num_ge() { + _a="$1" + _b="$2" + [ "$_a" != "-1" ] && [ -n "$_a" ] && awk -v a="$_a" -v b="$_b" 'BEGIN { exit (a + 0 >= b + 0) ? 0 : 1 }' +} + +watch_cooldown_active() { + [ -r "$WAN_WATCH_COOLDOWN_FILE" ] || return 1 + _last="$(cat "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null)" || return 1 + _now="$(date +%s 2>/dev/null)" || return 1 + [ "$((_now - _last))" -lt "$WAN_WATCH_COOLDOWN_S" ] 2>/dev/null +} + +watch_cooldown_stamp() { + date +%s > "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null || true +} + +# Returns trigger list (comma-separated) or empty. +detect_anomalies() { + _line="$1" + _role="$2" + _triggers="" + + for _pfx in wan fe be gw router; do + _avg="$(field_value "$_line" "${_pfx}_avg_ms")" + _loss="$(field_value "$_line" "${_pfx}_loss")" + _mdev="$(field_value "$_line" "${_pfx}_mdev_ms")" + _max="$(field_value "$_line" "${_pfx}_max_ms")" + _thr="$FE_ALERT_MS" + case "$_pfx" in + wan) _thr="$WAN_ALERT_MS" ;; + fe) _thr="$FE_ALERT_MS" ;; + be) _thr="$BE_ALERT_MS" ;; + gw) _thr="$GW_ALERT_MS" ;; + router) _thr="$ROUTER_ALERT_MS" ;; + esac + if num_ge "$_avg" "$_thr"; then + _triggers="${_triggers}${_triggers:+,}${_pfx}_avg=${_avg}" + fi + if num_ge "$_loss" "$WAN_LOSS_ALERT_PCT"; then + _triggers="${_triggers}${_triggers:+,}${_pfx}_loss=${_loss}" + fi + if num_ge "$_mdev" "$WAN_MDEV_ALERT_MS"; then + _triggers="${_triggers}${_triggers:+,}${_pfx}_mdev=${_mdev}" + fi + if num_ge "$_max" "$((_thr * 3))"; then + _triggers="${_triggers}${_triggers:+,}${_pfx}_max=${_max}" + fi + done + + _neigh="$(printf '%s\n' "$_line" | sed -n 's/.*neigh_fe=\([^ ]*\).*/\1/p')" + if [ -n "$_neigh" ] && [ "$_neigh" != "REACHABLE" ] && [ "$_neigh" != "DELAY" ]; then + _triggers="${_triggers}${_triggers:+,}neigh_fe=${_neigh}" + fi + + _ct_pct="$(printf '%s\n' "$_line" | sed -n 's/.*(\([0-9]*\)%).*/\1/p' | head -1)" + if [ -n "$_ct_pct" ] && num_ge "$_ct_pct" "$WAN_CONNTRACK_ALERT_PCT"; then + _triggers="${_triggers}${_triggers:+,}conntrack_pct=${_ct_pct}" + fi + + # Pi: WAN ok but FE slow -> path anomaly (even if FE just above threshold). + if [ "$_role" = "pi" ]; then + _wan_avg="$(field_value "$_line" "wan_avg_ms")" + _fe_avg="$(field_value "$_line" "fe_avg_ms")" + if num_ge "$_fe_avg" "$FE_ALERT_MS" && ! num_ge "$_wan_avg" "$WAN_ALERT_MS"; then + case "$_triggers" in *fe_avg=*) ;; *) + _triggers="${_triggers}${_triggers:+,}pi_fe_path" ;; + esac + fi + fi + + # Router: WAN bad, FE ok -> edge WAN leg (matches monstro xenbr0 pattern). + if [ "$_role" = "router" ]; then + _wan_avg="$(field_value "$_line" "wan_avg_ms")" + _fe_avg="$(field_value "$_line" "fe_avg_ms")" + if num_ge "$_wan_avg" "$WAN_ALERT_MS" && ! num_ge "$_fe_avg" "$FE_ALERT_MS"; then + case "$_triggers" in *wan_avg=*) ;; *) + _triggers="${_triggers}${_triggers:+,}router_wan_path" ;; + esac + fi + fi + + printf '%s' "$_triggers" +} + +maybe_deep_diag() { + _line="$1" + _ts="$2" + _role="$3" + _hostn="$4" + + [ "$WAN_SNAPSHOT_WATCH" = "1" ] || return 0 + [ "${WAN_DEEP_DIAG_LOADED:-0}" = "1" ] || { + echo "wan-snapshot: watch enabled but wan-deep-diag.sh not loaded (install alongside wan-snapshot.sh)" >&2 + return 0 + } + + _triggers="$(detect_anomalies "$_line" "$_role")" + [ -n "$_triggers" ] || return 0 + + if watch_cooldown_active; then + wan_deep_emit alert "$_ts" "$_role" "$_hostn" \ + "parent_ts=$_ts triggers=\"$_triggers\" action=skipped_cooldown" + return 0 + fi + + _gw_target="$GW_PING_TARGET" + if [ "$_gw_target" = "auto" ]; then + _gw_target="$(default_gw)" + fi + _wdev="$(route_dev "$WAN_PING_TARGET")" + _fdev="$(route_dev "$FE_PING_TARGET")" + + wan_deep_diag_run "$_ts" "$_role" "$_hostn" "$_triggers" \ + "$WAN_PING_TARGET" "$FE_PING_TARGET" "$_gw_target" "$_wdev" "$_fdev" + watch_cooldown_stamp +} + +append_watch_log() { + _line="$1" + _dir="$(dirname "$WAN_WATCH_LOG")" + [ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true + printf '%s\n' "$_line" >> "$WAN_WATCH_LOG" +} + acquire_lock() { command -v flock >/dev/null 2>&1 || return 0 _lock="$WAN_SNAPSHOT_LOCK" _dir="$(dirname "$_lock")" - if ! mkdir -p "$_dir" 2>/dev/null || ! : >"$_lock" 2>/dev/null; then - _lock="${TMPDIR:-/tmp}/wan-snapshot.$(id -u 2>/dev/null || echo 0).lock" + _fallback="${TMPDIR:-/tmp}/wan-snapshot.$(id -u 2>/dev/null || echo 0).lock" + if [ ! -d "$_dir" ] || [ ! -w "$_dir" ] 2>/dev/null; then + _lock="$_fallback" + elif ! ( : >>"$_lock" ) 2>/dev/null; then + _lock="$_fallback" fi exec 9>"$_lock" 2>/dev/null || return 0 flock -n 9 || { @@ -204,7 +365,7 @@ snapshot_once() { _role="$(detect_role)" _host="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)" - _line="ts=$_ts ver=2 role=$_role host=$_host" + _line="kind=snapshot ts=$_ts ver=2 role=$_role host=$_host" _line="$(append_ping_fields "$_line" "wan" "$WAN_PING_TARGET")" _line="$(append_ping_fields "$_line" "fe" "$FE_PING_TARGET")" @@ -241,9 +402,15 @@ snapshot_once() { [ -n "$_drops" ] && _line="$_line drops=\"$_drops\"" fi - _dir="$(dirname "$WAN_SNAPSHOT_LOG")" - [ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true - printf '%s\n' "$_line" >> "$WAN_SNAPSHOT_LOG" + append_watch_log "$_line" + if [ "$WAN_SNAPSHOT_LOG" != "$WAN_WATCH_LOG" ]; then + _dir="$(dirname "$WAN_SNAPSHOT_LOG")" + [ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true + printf '%s\n' "$_line" >> "$WAN_SNAPSHOT_LOG" + fi + + maybe_deep_diag "$_line" "$_ts" "$_role" "$_host" + printf '%s\n' "$_line" } @@ -271,7 +438,7 @@ day_stats() { _field="$3" [ -r "$_log" ] || return 0 awk -v d="$_date" -v f="$_field" ' - $0 ~ "^ts=" d { + ($0 ~ "^kind=snapshot ts=" d || ($0 ~ "^ts=" d && $0 !~ / kind=/)) { for (i = 1; i <= NF; i++) { if ($i ~ "^" f "=") { split($i, a, "=") @@ -302,7 +469,7 @@ day_max_loss() { _field="$3" [ -r "$_log" ] || return 0 awk -v d="$_date" -v f="$_field" ' - $0 ~ "^ts=" d { + ($0 ~ "^kind=snapshot ts=" d || ($0 ~ "^ts=" d && $0 !~ / kind=/)) { for (i = 1; i <= NF; i++) { if ($i ~ "^" f "=") { split($i, a, "=") @@ -323,7 +490,7 @@ report_day_block() { _log="$1" _label="$2" _date="$3" - _samples="$(grep -c "^ts=$_date" "$_log" 2>/dev/null || echo 0)" + _samples="$(grep -cE "^kind=snapshot ts=$_date|^ts=$_date" "$_log" 2>/dev/null || echo 0)" printf '%s=%s samples=%s\n' "$_label" "$_date" "$_samples" for _pfx in wan fe be router gw; do _stats="$(day_stats "$_log" "$_date" "${_pfx}_avg_ms")" @@ -344,7 +511,7 @@ report_day_block() { } report() { - _log="$WAN_SNAPSHOT_LOG" + _log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}" _today="$(date '+%Y-%m-%d')" _yesterday="$(date -d 'yesterday' '+%Y-%m-%d' 2>/dev/null || date -v-1d '+%Y-%m-%d' 2>/dev/null || echo '')" @@ -357,7 +524,7 @@ report() { printf 'log=%s role=%s\n' "$_log" "$_role" report_day_block "$_log" "today" "$_today" - if [ -n "$_yesterday" ] && grep -q "^ts=$_yesterday" "$_log" 2>/dev/null; then + if [ -n "$_yesterday" ] && grep -qE "^kind=snapshot ts=$_yesterday|^ts=$_yesterday" "$_log" 2>/dev/null; then report_day_block "$_log" "yesterday" "$_yesterday" fi @@ -373,11 +540,22 @@ report() { tail_log() { _n="${1:-24}" - [ -r "$WAN_SNAPSHOT_LOG" ] || { - echo "wan-snapshot: no log at $WAN_SNAPSHOT_LOG" >&2 + _log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}" + [ -r "$_log" ] || { + echo "wan-snapshot: no log at $_log" >&2 exit 1 } - tail -n "$_n" "$WAN_SNAPSHOT_LOG" + grep -E '^kind=snapshot |^ts=' "$_log" 2>/dev/null | tail -n "$_n" +} + +tail_alerts() { + _n="${1:-40}" + _log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}" + [ -r "$_log" ] || { + echo "wan-snapshot: no log at $_log" >&2 + exit 1 + } + grep -E '^kind=alert |^kind=deep ' "$_log" 2>/dev/null | tail -n "$_n" } # Guess role label for correlate when log line has no role=. @@ -428,11 +606,12 @@ correlate_logs() { } _label="$(basename "$_path")" while IFS= read -r _line || [ -n "$_line" ]; do + is_snapshot_line "$_line" || continue case "$_line" in - ts=*) ;; + kind=snapshot*) _ts="${_line#kind=snapshot ts=}" ;; + ts=*) _ts="${_line#ts=}" ;; *) continue ;; esac - _ts="${_line#ts=}" _ts="${_ts%% *}" _slot="$(slot_key "$_ts")" _role="$(log_role_label "$_path" "$_line")" @@ -500,6 +679,9 @@ case "${1:-}" in --tail|tail) tail_log "${2:-24}" ;; + --alerts|alerts) + tail_alerts "${2:-40}" + ;; --correlate|correlate) shift correlate_logs "$@"