#!/bin/sh # # wan-watch.sh — one script, one log, immediate verdict every run. # # Deploy (Pi + monstro + FE): # sudo cp wan-watch.sh /usr/local/bin/wan-watch.sh # sudo chmod +x /usr/local/bin/wan-watch.sh # sudo cp wan-watch.env.example /etc/conf.d/wan-watch # # Cron (every 5 min — fast enough to catch 15-min pulses): # */5 * * * * root . /etc/conf.d/wan-watch 2>/dev/null; /usr/local/bin/wan-watch.sh >>/var/log/wan-watch.cron.log 2>&1 # # Manual: # wan-watch.sh # cron snapshot + probes + verdict # wan-watch.sh --full # human-readable dump NOW (no wait) # wan-watch.sh --analyze # conclusion from log (last 24h default) # wan-watch.sh --timeline # hourly good/bad bars # wan-watch.sh --force-deep # full deep capture even if OK # set -eu WAN_WATCH_LOG="${WAN_WATCH_LOG:-/var/log/wan-watch.log}" WAN_WATCH_ROLE="${WAN_WATCH_ROLE:-${WAN_SNAPSHOT_ROLE:-auto}}" WAN_WATCH_LOCK="${WAN_WATCH_LOCK:-/var/run/wan-watch.lock}" WAN_WATCH_COOLDOWN_S="${WAN_WATCH_COOLDOWN_S:-300}" WAN_WATCH_COOLDOWN_FILE="${WAN_WATCH_COOLDOWN_FILE:-/var/run/wan-watch-deep.stamp}" WAN_WATCH_DEEP="${WAN_WATCH_DEEP:-auto}" WAN_ANALYZE_HOURS="${WAN_ANALYZE_HOURS:-24}" WAN_PING_TARGET="${WAN_PING_TARGET:-1.1.1.1}" FE_PING_TARGET="${FE_PING_TARGET:-10.7.0.10}" BE_PING_TARGET="${BE_PING_TARGET-10.7.16.128}" ROUTER_PING_TARGET="${ROUTER_PING_TARGET:-none}" GW_PING_TARGET="${GW_PING_TARGET:-auto}" PING_COUNT="${PING_COUNT:-3}" PING_DEADLINE_S="${PING_DEADLINE_S:-6}" DEEP_PING_COUNT="${DEEP_PING_COUNT:-8}" DEEP_PING_DEADLINE_S="${DEEP_PING_DEADLINE_S:-10}" WAN_ALERT_MS="${WAN_ALERT_MS:-50}" FE_ALERT_MS="${FE_ALERT_MS:-50}" BE_ALERT_MS="${BE_ALERT_MS:-50}" GW_ALERT_MS="${GW_ALERT_MS:-50}" ROUTER_ALERT_MS="${ROUTER_ALERT_MS:-50}" LOSS_ALERT_PCT="${LOSS_ALERT_PCT:-1}" MDEV_ALERT_MS="${MDEV_ALERT_MS:-80}" CONNTRACK_ALERT_PCT="${CONNTRACK_ALERT_PCT:-70}" # --------------------------------------------------------------------------- emit() { _kind="$1"; _ts="$2"; _role="$3"; _host="$4" shift 4 _line="kind=$_kind ts=$_ts ver=3 role=$_role host=$_host $*" _dir="$(dirname "$WAN_WATCH_LOG")" mkdir -p "$_dir" 2>/dev/null || true printf '%s\n' "$_line" >> "$WAN_WATCH_LOG" } detect_role() { _role="$WAN_WATCH_ROLE" if [ "$_role" != "auto" ]; then printf '%s' "$_role"; return; fi _hn="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)" case "$_hn" in *raspberry*|*rpi*) printf 'pi' ;; *monstro*|*router*|*gw*) printf 'router' ;; *gentoo*|*wg0*|*fe*) printf 'fe' ;; *) printf 'host' ;; esac } default_gw() { ip route show default 2>/dev/null | awk '{print $3; exit}'; } route_dev() { ip route get "$1" 2>/dev/null | awk '{for (i=1;i<=NF;i++) if ($i=="dev") {print $(i+1); exit}}' } route_via() { ip route get "$1" 2>/dev/null | awk '{for (i=1;i<=NF;i++) if ($i=="via") {print $(i+1); exit}}' } load_1m() { awk '{print $1}' /proc/loadavg 2>/dev/null || echo n/a; } cpu_busy_pct() { awk '/^cpu /{idle=$5+$6;t=0;for(i=2;i<=NF;i++)t+=$i;if(t>0)printf"%.0f",100*(t-idle)/t}' /proc/stat 2>/dev/null || echo n/a } conntrack_kv() { _c="$(cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null)" || _c="" _m="$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null)" || _m="" if [ -n "$_c" ] && [ -n "$_m" ] && [ "$_m" -gt 0 ] 2>/dev/null; then printf 'conntrack=%s/%s(%s%%)' "$_c" "$_m" "$((_c * 100 / _m))" fi } neigh_state() { ip neigh show "$1" 2>/dev/null | awk '{print $6; exit}'; } num_ge() { awk -v a="$1" -v b="$2" 'BEGIN{exit(a+0>=b+0)?0:1}' 2>/dev/null } ping_stats() { _host="$1"; _cnt="${2:-$PING_COUNT}"; _dead="${3:-$PING_DEADLINE_S}" _out="$(ping -c "$_cnt" -w "$_dead" "$_host" 2>/dev/null)" || _out="" if [ -z "$_out" ]; then printf '%s %s %s %s %s' -1 -1 -1 -1 -1; return; fi _loss="$(printf '%s\n' "$_out" | awk '/packet loss/{for(i=1;i<=NF;i++) if($i~/%$/){gsub(/%/,"",$i);print $i;exit}}')" _rtt="$(printf '%s\n' "$_out" | sed -n 's/.*= \([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\) ms.*/\1 \2 \3 \4/p' | head -1)" if [ -n "$_rtt" ]; then set -- $_rtt printf '%s %s %s %s %s' "${_loss:-0}" "$1" "$2" "$3" "$4" else _avg="$(printf '%s\n' "$_out" | awk -F'[=/ ]' '/min\/avg\/max/{print $8;exit}')" printf '%s %s %s %s %s' "${_loss:-0}" "${_avg:--1}" "${_avg:--1}" "${_avg:--1}" 0 fi } append_ping() { _line="$1"; _pfx="$2"; _host="$3" set -- $(ping_stats "$_host") _loss="$1"; _min="$2"; _avg="$3"; _max="$4"; _mdev="$5" printf '%s %s=%s %s_loss=%s%% %s_min_ms=%s %s_avg_ms=%s %s_max_ms=%s %s_mdev_ms=%s' \ "$_line" "$_pfx" "$_host" "$_pfx" "$_loss" "$_pfx" "$_min" "$_pfx" "$_avg" "$_pfx" "$_max" "$_pfx" "$_mdev" } field_val() { printf '%s\n' "$2" | awk -v f="$1" '{for(i=1;i<=NF;i++) if($i~"^"f"="){split($i,a,"=");gsub(/%/,"",a[2]);print a[2];exit}}' } acquire_lock() { command -v flock >/dev/null 2>&1 || return 0 _lock="$WAN_WATCH_LOCK" _fb="${TMPDIR:-/tmp}/wan-watch.$$.lock" _dir="$(dirname "$_lock")" if [ ! -w "$_dir" ] 2>/dev/null; then _lock="${TMPDIR:-/tmp}/wan-watch.$(id -u 2>/dev/null || echo 0).lock"; fi exec 9>"$_lock" 2>/dev/null || return 0 flock -n 9 || { echo "wan-watch: skip (running)" >&2; exit 0; } } cooldown_active() { [ -r "$WAN_WATCH_COOLDOWN_FILE" ] || return 1 _l="$(cat "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null)" || return 1 _n="$(date +%s 2>/dev/null)" || return 1 [ "$((_n - _l))" -lt "$WAN_WATCH_COOLDOWN_S" ] 2>/dev/null } # --------------------------------------------------------------------------- # Probes: tunnels, services, kernel — cheap, every run # --------------------------------------------------------------------------- probe_emit() { _ts="$1"; _role="$2"; _host="$3"; _section="$4" shift 4 emit probe "$_ts" "$_role" "$_host" "section=$_section $*" } probe_all() { _ts="$1"; _role="$2"; _host="$3" # Kernel modules (GRE/PPTP) if [ -r /proc/modules ]; then _gre="$(grep -E '^ip_gre |^gre ' /proc/modules 2>/dev/null | awk '{print $1}' | tr '\n' ',' | sed 's/,$//')" _pptp="$(grep -E 'nf_conntrack_pptp|nf_nat_pptp' /proc/modules 2>/dev/null | awk '{print $1}' | tr '\n' ',' | sed 's/,$//')" [ -n "$_gre" ] && probe_emit "$_ts" "$_role" "$_host" gre "loaded=${_gre}" [ -n "$_pptp" ] && probe_emit "$_ts" "$_role" "$_host" pptp_mod "loaded=${_pptp}" fi # GRE / gretap links ip -d link show type gre 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" ' /^[0-9]+:/{name=$2; gsub(":","",name); state=$0; getline; detail=$0; printf "kind=probe ts=%s ver=3 role=%s host=%s section=gre_iface name=%s up=%s\n", ts,role,host,name,(index(state,"UP")?"yes":"no")} ' >> "$WAN_WATCH_LOG" 2>/dev/null || true # PPTP listeners / sessions _p1723="$(ss -lnpt 2>/dev/null | grep -c ':1723 ' || echo 0)" _pppd="$(pgrep -c pppd 2>/dev/null || echo 0)" _pptpd="$(pgrep -c pptpd 2>/dev/null || echo 0)" probe_emit "$_ts" "$_role" "$_host" pptp "listen1723=${_p1723} pppd=${_pppd} pptpd=${_pptpd}" # tinc _tinc="$(pgrep -c tincd 2>/dev/null || echo 0)" if [ "$_tinc" -gt 0 ] 2>/dev/null; then _tn="" if command -v tincctl >/dev/null 2>&1; then for _net in /etc/tinc/*/tinc.conf; do [ -f "$_net" ] || continue _n="$(basename "$(dirname "$_net")")" _st="$(tincctl -n "$_n" status 2>/dev/null | tr '\n' ' ' | sed 's/ */ /g')" probe_emit "$_ts" "$_role" "$_host" tinc "net=${_n} tincd=${_tinc} status=${_st:-unknown}" done else probe_emit "$_ts" "$_role" "$_host" tinc "tincd=${_tinc} status=no_tincctl" fi else probe_emit "$_ts" "$_role" "$_host" tinc "tincd=0" fi # WireGuard if command -v wg >/dev/null 2>&1; then wg show all 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" ' /^interface:/ { iface=$2; peer=""; hs=""; ep=""; tx=""; rx="" } /^ public key:/ { pk=$3 } /^ endpoint:/ { ep=$2 } /^ latest handshake:/ { hs=$0; sub(/^ latest handshake: /,"",hs) } /^ transfer:/ { tx=$2; rx=$4 } /^$/ { if (iface != "") printf "kind=probe ts=%s ver=3 role=%s host=%s section=wireguard iface=%s endpoint=%s handshake=%s tx=%s rx=%s\n", ts, role, host, iface, ep, hs, tx, rx } END { if (iface != "") printf "kind=probe ts=%s ver=3 role=%s host=%s section=wireguard iface=%s endpoint=%s handshake=%s tx=%s rx=%s\n", ts, role, host, iface, ep, hs, tx, rx } ' >> "$WAN_WATCH_LOG" 2>/dev/null || true fi # iodine / dns tunnel iface _iod="$(pgrep -c iodined 2>/dev/null || echo 0)" probe_emit "$_ts" "$_role" "$_host" iodine "iodined=${_iod}" for _d in dns0 tun0 tap0; do ip link show "$_d" >/dev/null 2>&1 || continue ip -s link show "$_d" 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v d="$_d" ' /RX:/{getline; re=$3; rd=$4} /TX:/{getline; te=$3; td=$4} END{printf "kind=probe ts=%s ver=3 role=%s host=%s section=tunnel_iface dev=%s rx_err=%s rx_drop=%s tx_err=%s tx_drop=%s\n", ts,role,host,d,re+0,rd+0,te+0,td+0} ' >> "$WAN_WATCH_LOG" 2>/dev/null || true done # Named / squid / nginx (edge services) for _svc in named bind9 squid nginx iodined pptpd; do _c="$(pgrep -c "$_svc" 2>/dev/null || echo 0)" [ "$_c" -gt 0 ] 2>/dev/null || continue _cpu="$(ps -eo pcpu,comm 2>/dev/null | awk -v s="$_svc" '$2~s{print $1; exit}')" probe_emit "$_ts" "$_role" "$_host" service "name=${_svc} procs=${_c} cpu=${_cpu:-n/a}" done # Key ifaces stats for _d in xenbr0 eno0 eth0 wlan0 br0; do ip link show "$_d" >/dev/null 2>&1 || continue ip -s link show "$_d" 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v d="$_d" ' /RX:/{getline; re=$3; rd=$4} /TX:/{getline; te=$3; td=$4} END{printf "kind=probe ts=%s ver=3 role=%s host=%s section=iface dev=%s rx_drop=%s tx_drop=%s rx_err=%s tx_err=%s\n", ts,role,host,d,rd+0,td+0,re+0,te+0} ' >> "$WAN_WATCH_LOG" 2>/dev/null || true tc -s qdisc show dev "$_d" 2>/dev/null | head -3 | while IFS= read -r _ql; do probe_emit "$_ts" "$_role" "$_host" qdisc "dev=${_d} detail=${_ql}" done done # softnet drops if [ -r /proc/net/softnet_stat ]; then awk -v ts="$_ts" -v role="$_role" -v host="$_host" '$2+0>0{ printf "kind=probe ts=%s ver=3 role=%s host=%s section=softnet cpu=%d dropped=%s\n",ts,role,host,NR-1,$2 }' /proc/net/softnet_stat >> "$WAN_WATCH_LOG" 2>/dev/null || true fi # Routes + neigh for key targets for _label in wan:$WAN_PING_TARGET fe:$FE_PING_TARGET gw:${GW_PING_TARGET:-auto}; do _l="${_label%%:*}"; _t="${_label#*:}" [ "$_t" = "auto" ] && _t="$(default_gw)" [ -n "$_t" ] && [ "$_t" != none ] || continue ip route get "$_t" 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v l="$_l" -v t="$_t" ' {dev=via=src=""; for(i=1;i<=NF;i++){if($i=="dev")dev=$(i+1);if($i=="via")via=$(i+1);if($i=="src")src=$(i+1)} printf "kind=probe ts=%s ver=3 role=%s host=%s section=route label=%s target=%s dev=%s via=%s src=%s\n",ts,role,host,l,t,dev,via,src} ' >> "$WAN_WATCH_LOG" 2>/dev/null || true done _ns="$(neigh_state "$FE_PING_TARGET")" [ -n "$_ns" ] && probe_emit "$_ts" "$_role" "$_host" neigh "ip=${FE_PING_TARGET} state=${_ns}" # iptables hot rules (router/fe) if command -v iptables >/dev/null 2>&1 && [ "$_role" = "router" ]; then for _spec in filter:FORWARD nat:PREROUTING nat:POSTROUTING; do _tbl="${_spec%%:*}"; _ch="${_spec#*:}" iptables -t "$_tbl" -L "$_ch" -n -v --line-numbers 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v tbl="$_tbl" -v ch="$_ch" ' /^[0-9]+/ && $2+0>1000 { printf "kind=probe ts=%s ver=3 role=%s host=%s section=iptables tbl=%s chain=%s line=%s pkts=%s target=%s\n", ts,role,host,tbl,ch,$1,$2,$4 }' | head -6 >> "$WAN_WATCH_LOG" 2>/dev/null || true done fi # Top CPU ps -eo pcpu=,pmem=,comm= --sort=-pcpu 2>/dev/null | head -5 | awk -v ts="$_ts" -v role="$_role" -v host="$_host" ' NF>=3{printf "kind=probe ts=%s ver=3 role=%s host=%s section=proc rank=%d cpu=%s mem=%s comm=%s\n",ts,role,host,NR,$1,$2,$3} ' >> "$WAN_WATCH_LOG" 2>/dev/null || true ss -s 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" '/TCP:|UDP:/{ gsub(/^[ \t]+/,""); printf "kind=probe ts=%s ver=3 role=%s host=%s section=ss %s\n",ts,role,host,$0 }' >> "$WAN_WATCH_LOG" 2>/dev/null || true } # --------------------------------------------------------------------------- # Deep capture (on anomaly) # --------------------------------------------------------------------------- deep_burst_line() { _ts="$1"; _role="$2"; _host="$3"; _label="$4"; _target="$5" set -- $(ping_stats "$_target" "$DEEP_PING_COUNT" "$DEEP_PING_DEADLINE_S") emit deep "$_ts" "$_role" "$_host" \ "parent_ts=$_ts section=burst label=$_label target=$_target loss=$1% min_ms=$2 avg_ms=$3 max_ms=$4 mdev_ms=$5" } deep_capture() { _ts="$1"; _role="$2"; _host="$3"; _triggers="$4" emit alert "$_ts" "$_role" "$_host" "parent_ts=$_ts triggers=\"$_triggers\"" _ct="$(conntrack_kv)" emit deep "$_ts" "$_role" "$_host" "parent_ts=$_ts section=host load1=$(load_1m) cpu_busy=$(cpu_busy_pct)% mem_avail_kb=$(awk '/^MemAvailable:/{print $2}' /proc/meminfo 2>/dev/null || echo n/a) ${_ct}" deep_burst_line "$_ts" "$_role" "$_host" wan "$WAN_PING_TARGET" deep_burst_line "$_ts" "$_role" "$_host" fe "$FE_PING_TARGET" _gw="$GW_PING_TARGET"; [ "$_gw" = auto ] && _gw="$(default_gw)" [ -n "$_gw" ] && [ "$_gw" != none ] && deep_burst_line "$_ts" "$_role" "$_host" gw "$_gw" probe_all "$_ts" "$_role" "$_host" } detect_triggers() { _line="$1"; _role="$2"; _t="" for _p in wan fe be gw router; do _a="$(field_val "${_p}_avg_ms" "$_line")" _l="$(field_val "${_p}_loss" "$_line")" _m="$(field_val "${_p}_mdev_ms" "$_line")" _thr="$FE_ALERT_MS" case "$_p" in wan) _thr="$WAN_ALERT_MS";; fe) _thr="$FE_ALERT_MS";; be) _thr="$BE_ALERT_MS";; gw) _thr="$GW_ALERT_MS";; router) _thr="$ROUTER_ALERT_MS";; esac num_ge "$_a" "$_thr" && _t="${_t}${_t:+,}${_p}_avg=${_a}" num_ge "$_l" "$LOSS_ALERT_PCT" && _t="${_t}${_t:+,}${_p}_loss=${_l}" num_ge "$_m" "$MDEV_ALERT_MS" && _t="${_t}${_t:+,}${_p}_mdev=${_m}" done _ct="$(printf '%s\n' "$_line" | sed -n 's/.*(\([0-9]*\)%).*/\1/p' | head -1)" num_ge "$_ct" "$CONNTRACK_ALERT_PCT" && _t="${_t}${_t:+,}conntrack_pct=${_ct}" _wan="$(field_val wan_avg_ms "$_line")"; _fe="$(field_val fe_avg_ms "$_line")" if [ "$_role" = pi ] && num_ge "$_fe" "$FE_ALERT_MS" && ! num_ge "$_wan" "$WAN_ALERT_MS"; then case "$_t" in *fe_avg=*) ;; *) _t="${_t}${_t:+,}pi_fe_path";; esac fi if [ "$_role" = router ] && num_ge "$_wan" "$WAN_ALERT_MS" && ! num_ge "$_fe" "$FE_ALERT_MS"; then case "$_t" in *wan_avg=*) ;; *) _t="${_t}${_t:+,}router_wan_path";; esac fi printf '%s' "$_t" } # --------------------------------------------------------------------------- # Verdict engine — runs EVERY time on current snapshot # --------------------------------------------------------------------------- compute_verdict() { _line="$1"; _role="$2" _probe_hint="${3:-}" _wan="$(field_val wan_avg_ms "$_line")" _fe="$(field_val fe_avg_ms "$_line")" _be="$(field_val be_avg_ms "$_line")" _gw="$(field_val gw_avg_ms "$_line")" _wmdev="$(field_val wan_mdev_ms "$_line")" _ct="$(printf '%s\n' "$_line" | sed -n 's/.*(\([0-9]*\)%).*/\1/p' | head -1)" _verdict="OK" _conf="high" _summary="All probes within thresholds." _action="none" if [ "$_role" = router ]; then if num_ge "$_wan" "$WAN_ALERT_MS" && ! num_ge "$_fe" 5; then _verdict="MONSTRO_WAN_BLOAT" _summary="WAN leg slow (~${_wan}ms) but FE local OK (~${_fe}ms). Edge xenbr0/upstream queue." _action="SQM/fq_codel on xenbr0; check gw $(route_via "$WAN_PING_TARGET" 2>/dev/null || echo 10.7.16.228); review squid/named load" elif num_ge "$_gw" "$GW_ALERT_MS" && num_ge "$_wan" "$WAN_ALERT_MS"; then _verdict="UPSTREAM_GW" _summary="WAN (${_wan}ms) and gw (${_gw}ms) both high — parent hop 10.7.16.228 stressed." _action="Check upstream link; correlate with ISP/gateway admin" elif num_ge "$_ct" "$CONNTRACK_ALERT_PCT"; then _verdict="CONNTRACK_PRESSURE" _summary="Conntrack at ${_ct}% — new flows may stall." _action="Inspect conntrack count; reduce NAT/tunnel sessions or raise nf_conntrack_max" elif num_ge "$_wmdev" "$MDEV_ALERT_MS"; then _verdict="WAN_JITTER" _summary="WAN jitter high (mdev=${_wmdev}ms) avg=${_wan}ms." _action="Bufferbloat likely; enable fq_codel" fi elif [ "$_role" = pi ]; then if num_ge "$_fe" "$FE_ALERT_MS" && ! num_ge "$_wan" "$WAN_ALERT_MS"; then _verdict="PATH_TO_FE" _summary="Pi ISP OK (~${_wan}ms) but FE path slow (~${_fe}ms). Problem at monstro edge, not Pi WiFi." _action="Check monstro wan-watch verdict; your traffic crosses modem WAN queue" elif num_ge "$_wan" "$WAN_ALERT_MS"; then _verdict="PI_ISP" _summary="Pi WAN to 1.1.1.1 slow (${_wan}ms)." _action="Local ISP/WiFi issue on Pi side" fi elif [ "$_role" = fe ]; then if num_ge "$_wan" "$WAN_ALERT_MS"; then _verdict="FE_WAN_EGRESS" _summary="FE WAN egress slow (${_wan}ms)." _action="Check FE default route and nginx upstream separately" fi fi # Probe hints from same run (grep log tail — set by caller in _probe_hint) case "${_probe_hint:-}" in *dns0*tx_drop=*|*tunnel_iface*dns0*) if [ "$_verdict" = "OK" ] || [ "$_verdict" = "MONSTRO_WAN_BLOAT" ]; then _verdict="TUNNEL_DNS0_LOAD" _summary="${_summary} iodine/dns0 tunnel drops elevated." _action="${_action}; inspect iodined and dns0 tx_drop" _conf="medium" fi ;; *named*cpu=*) if [ "$_verdict" = "MONSTRO_WAN_BLOAT" ] || [ "$_verdict" = "OK" ]; then _summary="${_summary} named DNS active on router." _conf="medium" fi ;; esac printf 'verdict=%s confidence=%s summary="%s" action="%s" wan_ms=%s fe_ms=%s gw_ms=%s' \ "$_verdict" "$_conf" "$_summary" "$_action" "$_wan" "$_fe" "$_gw" } # --------------------------------------------------------------------------- # Main cron run # --------------------------------------------------------------------------- run_watch() { command -v ping >/dev/null 2>&1 || { echo "wan-watch: need ping" >&2; exit 1; } acquire_lock _ts="$(date -Iseconds 2>/dev/null || date '+%Y-%m-%dT%H:%M:%S%z')" _role="$(detect_role)" _host="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)" _gw="$GW_PING_TARGET"; [ "$_gw" = auto ] && _gw="$(default_gw)" _ct="$(conntrack_kv)" _wdev="$(route_dev "$WAN_PING_TARGET")"; _fdev="$(route_dev "$FE_PING_TARGET")" _wgw="$(route_via "$WAN_PING_TARGET")" _ns="$(neigh_state "$FE_PING_TARGET")" _snap="$(append_ping "" wan "$WAN_PING_TARGET")" _snap="$(append_ping "$_snap" fe "$FE_PING_TARGET")" [ -n "$BE_PING_TARGET" ] && [ "$BE_PING_TARGET" != none ] && _snap="$(append_ping "$_snap" be "$BE_PING_TARGET")" [ -n "$ROUTER_PING_TARGET" ] && [ "$ROUTER_PING_TARGET" != none ] && _snap="$(append_ping "$_snap" router "$ROUTER_PING_TARGET")" [ -n "$_gw" ] && [ "$_gw" != none ] && _snap="$(append_ping "$_snap" gw "$_gw")" _snap="$_snap load1=$(load_1m) cpu_busy=$(cpu_busy_pct)%" [ -n "$_ct" ] && _snap="$_snap $_ct" [ -n "$_wdev" ] && _snap="$_snap route_wan=$_wdev" [ -n "$_fdev" ] && _snap="$_snap route_fe=$_fdev" [ -n "$_wgw" ] && _snap="$_snap via_wan=$_wgw" [ -n "$_ns" ] && _snap="$_snap neigh_fe=$_ns" emit snapshot "$_ts" "$_role" "$_host" "$_snap" >/dev/null probe_all "$_ts" "$_role" "$_host" _probe_hint="$(tail -30 "$WAN_WATCH_LOG" 2>/dev/null | grep 'kind=probe' | tr '\n' ' ')" _verdict_kv="$(compute_verdict "$_snap" "$_role" "$_probe_hint")" _verdict_line="kind=verdict ts=$_ts ver=3 role=$_role host=$_host $_verdict_kv" printf '%s\n' "$_verdict_line" >> "$WAN_WATCH_LOG" _triggers="$(detect_triggers "$_snap" "$_role")" _do_deep=0 if [ -n "$_triggers" ]; then _do_deep=1; fi if [ "$WAN_WATCH_DEEP" = "always" ]; then _do_deep=1; fi if [ "$WAN_WATCH_DEEP" = "never" ]; then _do_deep=0; fi if [ "$_do_deep" = 1 ]; then if cooldown_active && [ "$WAN_WATCH_DEEP" != "always" ]; then emit alert "$_ts" "$_role" "$_host" "parent_ts=$_ts triggers=\"$_triggers\" action=skipped_cooldown" >/dev/null else deep_capture "$_ts" "$_role" "$_host" "$_triggers" date +%s > "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null || true fi fi # Human one-liner to cron log printf 'WAN-WATCH [%s/%s] %s\n' "$_role" "$_host" "$_verdict_kv" } # --------------------------------------------------------------------------- # --analyze: final conclusion from log without waiting # --------------------------------------------------------------------------- analyze_log() { _log="$WAN_WATCH_LOG" _hours="${1:-$WAN_ANALYZE_HOURS}" [ -r "$_log" ] || { echo "wan-watch: no log $_log" >&2; exit 1; } _cut="$(date -d "${_hours} hours ago" '+%Y-%m-%dT%H:%M' 2>/dev/null || echo "")" echo "=== wan-watch analyze (last ${_hours}h) ===" echo "log=$_log" echo "" awk -v cut="$_cut" ' function ts_ok(t) { return (cut == "" || t >= cut) } /^kind=snapshot / && ts_ok($0) { for(i=1;i<=NF;i++) { if($i~/^wan_avg_ms=/){split($i,a,"="); wa=a[2]+0; wn++} if($i~/^fe_avg_ms=/){split($i,a,"="); fa=a[2]+0; fn++} if($i~/^gw_avg_ms=/){split($i,a,"="); ga=a[2]+0; gn++} if($i~/^role=/){split($i,a,"="); role=a[2]} } if(wa>=50){wb++} else if(wa>=0){wg++} if(fa>=50){fb++} else if(fa>=0){fg++} if(wa>=50 && fa<5) rw_ok_fe++ # router pattern if(fa>=50 && wa<30) pi_bad++ # pi pattern sn++ } /^kind=verdict / && ts_ok($0) { if($0~/verdict=OK/) okv++; else badv++ for(i=1;i<=NF;i++) if($i~/^verdict=/){split($i,a,"="); vc[a[2]]++} } /^kind=probe / && /section=tunnel_iface/ && /tx_drop=[1-9]/ { dns_drop++ } /^kind=probe / && /section=service/ && /name=named/ { named++ } /^kind=probe / && /section=service/ && /name=squid/ { squid++ } END { printf "snapshots=%d (wan_bad=%d wan_good=%d fe_bad=%d fe_good=%d)\n", sn, wb+0, wg+0, fb+0, fg+0 printf "router_wan_bad_fe_local_ok=%d (MONSTRO_WAN_BLOAT signature)\n", rw_ok_fe+0 printf "pi_fe_bad_wan_ok=%d (PATH_TO_FE signature)\n", pi_bad+0 printf "verdicts: ok=%d anomaly=%d\n", okv+0, badv+0 printf "verdict_counts:" for(v in vc) printf " %s=%d", v, vc[v] printf "\n" printf "probe_hints: dns0_tx_drop_events=%d named_samples=%d squid_samples=%d\n", dns_drop+0, named+0, squid+0 pct = (sn>0) ? int(100*(wb+fb)/(2*sn)) : 0 printf "\n" if(rw_ok_fe>=3 || pi_bad>=3) { printf "CONCLUSION: MONSTRO WAN EDGE QUEUEING (confidence=HIGH)\n" printf " Evidence: wan slow + fe local OK; Pi ISP stays ~9ms in prior data.\n" printf " Not: FE VM, BE, Pi WiFi, global ISP.\n" printf " Fix order: 1) fq_codel/SQM on xenbr0 2) gw 10.7.16.228 3) squid/named 4) iodine dns0\n" } else if(wb+fb==0) { printf "CONCLUSION: HEALTHY (confidence=HIGH)\n" } else { printf "CONCLUSION: MIXED/INTERMITTENT (%d%% bad samples) — keep */5 cron running\n", pct } } ' "$_log" echo "" echo "=== last 5 verdicts ===" grep '^kind=verdict ' "$_log" 2>/dev/null | tail -5 | while IFS= read -r _l; do _t="$(printf '%s\n' "$_l" | sed 's/.*ts=\([^ ]*\).*/\1/')" _v="$(printf '%s\n' "$_l" | sed 's/.*verdict=\([^ ]*\).*/\1/')" _s="$(printf '%s\n' "$_l" | sed 's/.*summary="\([^"]*\)".*/\1/')" printf ' %s %s %s\n' "$_t" "$_v" "$_s" done } timeline_log() { _log="$WAN_WATCH_LOG" [ -r "$_log" ] || { echo "no log"; exit 1; } echo "=== hourly (snapshot wan/fe bad if avg>=50) ===" awk ' /^kind=snapshot / { ts=$0; sub(/.*ts=/,"",ts); sub(/ .*/,"",ts) hour=substr(ts,1,13) for(i=1;i<=NF;i++){ if($i~/^wan_avg_ms=/){split($i,a,"="); w=a[2]+0} if($i~/^fe_avg_ms=/){split($i,a,"="); f=a[2]+0} } t[hour]++; if(w>=50||f>=50) b[hour]++ } END { for(h in t){ bh=b[h]+0; printf " %s %d/%d bad ", h, bh, t[h] for(i=0;i/dev/null || echo "fail" done echo "--- host ---" echo "load=$(load_1m) cpu=$(cpu_busy_pct)% $(conntrack_kv)" echo "--- probes ---" probe_all "$_ts" "$_role" "$_host" tail -40 "$WAN_WATCH_LOG" | grep "^kind=probe " echo "--- verdict ---" _snap="ts=$_ts role=$_role host=$_host" _snap="$(append_ping "$_snap" wan "$WAN_PING_TARGET")" _snap="$(append_ping "$_snap" fe "$FE_PING_TARGET")" compute_verdict "$_snap" "$_role" echo "" } correlate_logs() { if [ "$#" -lt 1 ]; then echo "wan-watch: --correlate needs log files" >&2; exit 1; fi _tmp="$(mktemp "${TMPDIR:-/tmp}/wan-corr.XXXXXX")" trap 'rm -f "$_tmp"' EXIT INT HUP for _path in "$@"; do [ -r "$_path" ] || continue while IFS= read -r _line || [ -n "$_line" ]; do case "$_line" in kind=snapshot*) _ts="${_line#*ts=}"; _ts="${_ts%% *}" ;; ts=*) _ts="${_line#ts=}"; _ts="${_ts%% *}" ;; *) continue ;; esac _date="${_ts%%T*}"; _rest="${_ts#*T}"; _hour="${_rest%%:*}" _min="${_rest#*:}"; _min="${_min%%:*}"; _min="${_min%%+*}"; _min="${_min%%-*}" _slot="${_date}T${_hour}:$(printf '%02d' $((_min / 15 * 15)))" _role="$(printf '%s\n' "$_line" | awk '{for(i=1;i<=NF;i++) if($i~/^role=/){split($i,a,"=");print a[2];exit}}')" [ -n "$_role" ] || _role="$(basename "$_path" .log)" printf '%s\t%s\t%s\t%s\n' "$_slot" "$_role" "$_ts" "$_line" >> "$_tmp" done < "$_path" done printf 'slot\t' _roles="$(awk -F'\t' '{print $2}' "$_tmp" | sort -u | tr '\n' ' ')" for _r in $_roles; do printf '%s_wan\t%s_fe\t%s_verdict\t' "$_r" "$_r" "$_r"; done printf '\n' awk -F'\t' ' { slot=$1; role=$2; ts=$3; line=$4; key=slot SUBSEP role if (!(key in best) || ts>best[key]) { best[key]=ts; wan=fe="-"; ver="-" for(i=1;i<=split(line,a," ");i++){ if(a[i]~/^wan_avg_ms=/){split(a[i],b,"=");wan=b[2]} if(a[i]~/^fe_avg_ms=/){split(a[i],b,"=");fe=b[2]} } W[slot,role,"wan"]=wan; W[slot,role,"fe"]=fe slots[slot]=1; roles[role]=1 } } END{ nr=0; for(r in roles) rl[++nr]=r; asort(rl) ns=0; for(s in slots) sl[++ns]=s; asort(sl) for(i=1;i<=ns;i++){ s=sl[i]; printf "%s", s for(j=1;j<=nr;j++){ r=rl[j]; printf "\t%s\t%s\t-", W[s,r,"wan"]+0?W[s,r,"wan"]:"-", W[s,r,"fe"]+0?W[s,r,"fe"]:"-" } printf "\n" } } ' "$_tmp" } usage() { cat </dev/null | tail -n "$_n" || true ;; "") run_watch ;; *) echo "wan-watch: unknown $1"; usage; exit 1 ;; esac