ac/android_cast
1
0
Fork 0
mirror of git://f0xx.org/android_cast synced 2026-07-29 07:20:00 +03:00
Code Issues Packages Projects Releases Wiki Activity
Files
9a834114169d0ed0a35690b6fc9747b3ce3e1f1a
android_cast/examples/network/wan-watch.sh
Anton Afanasyeu 86055bf509 sync on be, vpn
2026-06-13 15:31:18 +02:00

672 lines
28 KiB
Bash
Executable File
Raw Blame History

#!/bin/sh
#
# wan-watch.sh — one script, one log, immediate verdict every run.
#
# Deploy (Pi + monstro + FE):
# sudo cp wan-watch.sh /usr/local/bin/wan-watch.sh
# sudo chmod +x /usr/local/bin/wan-watch.sh
# sudo cp wan-watch.env.example /etc/conf.d/wan-watch
#
# Cron (every 5 min — fast enough to catch 15-min pulses):
# */5 * * * * root . /etc/conf.d/wan-watch 2>/dev/null; /usr/local/bin/wan-watch.sh >>/var/log/wan-watch.cron.log 2>&1
#
# Manual:
# wan-watch.sh # cron snapshot + probes + verdict
# wan-watch.sh --full # human-readable dump NOW (no wait)
# wan-watch.sh --analyze # conclusion from log (last 24h default)
# wan-watch.sh --timeline # hourly good/bad bars
# wan-watch.sh --force-deep # full deep capture even if OK
#
set -eu
WAN_WATCH_LOG="${WAN_WATCH_LOG:-/var/log/wan-watch.log}"
WAN_WATCH_ROLE="${WAN_WATCH_ROLE:-${WAN_SNAPSHOT_ROLE:-auto}}"
WAN_WATCH_LOCK="${WAN_WATCH_LOCK:-/var/run/wan-watch.lock}"
WAN_WATCH_COOLDOWN_S="${WAN_WATCH_COOLDOWN_S:-300}"
WAN_WATCH_COOLDOWN_FILE="${WAN_WATCH_COOLDOWN_FILE:-/var/run/wan-watch-deep.stamp}"
WAN_WATCH_DEEP="${WAN_WATCH_DEEP:-auto}"
WAN_ANALYZE_HOURS="${WAN_ANALYZE_HOURS:-24}"
WAN_PING_TARGET="${WAN_PING_TARGET:-1.1.1.1}"
FE_PING_TARGET="${FE_PING_TARGET:-10.7.0.10}"
BE_PING_TARGET="${BE_PING_TARGET-10.7.16.128}"
ROUTER_PING_TARGET="${ROUTER_PING_TARGET:-none}"
GW_PING_TARGET="${GW_PING_TARGET:-auto}"
PING_COUNT="${PING_COUNT:-3}"
PING_DEADLINE_S="${PING_DEADLINE_S:-6}"
DEEP_PING_COUNT="${DEEP_PING_COUNT:-8}"
DEEP_PING_DEADLINE_S="${DEEP_PING_DEADLINE_S:-10}"
WAN_ALERT_MS="${WAN_ALERT_MS:-50}"
FE_ALERT_MS="${FE_ALERT_MS:-50}"
BE_ALERT_MS="${BE_ALERT_MS:-50}"
GW_ALERT_MS="${GW_ALERT_MS:-50}"
ROUTER_ALERT_MS="${ROUTER_ALERT_MS:-50}"
LOSS_ALERT_PCT="${LOSS_ALERT_PCT:-1}"
MDEV_ALERT_MS="${MDEV_ALERT_MS:-80}"
CONNTRACK_ALERT_PCT="${CONNTRACK_ALERT_PCT:-70}"
# ---------------------------------------------------------------------------
emit() {
_kind="$1"; _ts="$2"; _role="$3"; _host="$4"
shift 4
_line="kind=$_kind ts=$_ts ver=3 role=$_role host=$_host $*"
_dir="$(dirname "$WAN_WATCH_LOG")"
mkdir -p "$_dir" 2>/dev/null || true
printf '%s\n' "$_line" >> "$WAN_WATCH_LOG"
}
detect_role() {
_role="$WAN_WATCH_ROLE"
if [ "$_role" != "auto" ]; then printf '%s' "$_role"; return; fi
_hn="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)"
case "$_hn" in
*raspberry*|*rpi*) printf 'pi' ;;
*monstro*|*router*|*gw*) printf 'router' ;;
*gentoo*|*wg0*|*fe*) printf 'fe' ;;
*) printf 'host' ;;
esac
}
default_gw() { ip route show default 2>/dev/null | awk '{print $3; exit}'; }
route_dev() {
ip route get "$1" 2>/dev/null | awk '{for (i=1;i<=NF;i++) if ($i=="dev") {print $(i+1); exit}}'
}
route_via() {
ip route get "$1" 2>/dev/null | awk '{for (i=1;i<=NF;i++) if ($i=="via") {print $(i+1); exit}}'
}
load_1m() { awk '{print $1}' /proc/loadavg 2>/dev/null || echo n/a; }
cpu_busy_pct() {
awk '/^cpu /{idle=$5+$6;t=0;for(i=2;i<=NF;i++)t+=$i;if(t>0)printf"%.0f",100*(t-idle)/t}' /proc/stat 2>/dev/null || echo n/a
}
conntrack_kv() {
_c="$(cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null)" || _c=""
_m="$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null)" || _m=""
if [ -n "$_c" ] && [ -n "$_m" ] && [ "$_m" -gt 0 ] 2>/dev/null; then
printf 'conntrack=%s/%s(%s%%)' "$_c" "$_m" "$((_c * 100 / _m))"
fi
}
neigh_state() { ip neigh show "$1" 2>/dev/null | awk '{print $6; exit}'; }
num_ge() {
awk -v a="$1" -v b="$2" 'BEGIN{exit(a+0>=b+0)?0:1}' 2>/dev/null
}
ping_stats() {
_host="$1"; _cnt="${2:-$PING_COUNT}"; _dead="${3:-$PING_DEADLINE_S}"
_out="$(ping -c "$_cnt" -w "$_dead" "$_host" 2>/dev/null)" || _out=""
if [ -z "$_out" ]; then printf '%s %s %s %s %s' -1 -1 -1 -1 -1; return; fi
_loss="$(printf '%s\n' "$_out" | awk '/packet loss/{for(i=1;i<=NF;i++) if($i~/%$/){gsub(/%/,"",$i);print $i;exit}}')"
_rtt="$(printf '%s\n' "$_out" | sed -n 's/.*= \([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\) ms.*/\1 \2 \3 \4/p' | head -1)"
if [ -n "$_rtt" ]; then
set -- $_rtt
printf '%s %s %s %s %s' "${_loss:-0}" "$1" "$2" "$3" "$4"
else
_avg="$(printf '%s\n' "$_out" | awk -F'[=/ ]' '/min\/avg\/max/{print $8;exit}')"
printf '%s %s %s %s %s' "${_loss:-0}" "${_avg:--1}" "${_avg:--1}" "${_avg:--1}" 0
fi
}
append_ping() {
_line="$1"; _pfx="$2"; _host="$3"
set -- $(ping_stats "$_host")
_loss="$1"; _min="$2"; _avg="$3"; _max="$4"; _mdev="$5"
printf '%s %s=%s %s_loss=%s%% %s_min_ms=%s %s_avg_ms=%s %s_max_ms=%s %s_mdev_ms=%s' \
"$_line" "$_pfx" "$_host" "$_pfx" "$_loss" "$_pfx" "$_min" "$_pfx" "$_avg" "$_pfx" "$_max" "$_pfx" "$_mdev"
}
field_val() {
printf '%s\n' "$2" | awk -v f="$1" '{for(i=1;i<=NF;i++) if($i~"^"f"="){split($i,a,"=");gsub(/%/,"",a[2]);print a[2];exit}}'
}
acquire_lock() {
command -v flock >/dev/null 2>&1 || return 0
_lock="$WAN_WATCH_LOCK"
_fb="${TMPDIR:-/tmp}/wan-watch.$$.lock"
_dir="$(dirname "$_lock")"
if [ ! -w "$_dir" ] 2>/dev/null; then _lock="${TMPDIR:-/tmp}/wan-watch.$(id -u 2>/dev/null || echo 0).lock"; fi
exec 9>"$_lock" 2>/dev/null || return 0
flock -n 9 || { echo "wan-watch: skip (running)" >&2; exit 0; }
}
cooldown_active() {
[ -r "$WAN_WATCH_COOLDOWN_FILE" ] || return 1
_l="$(cat "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null)" || return 1
_n="$(date +%s 2>/dev/null)" || return 1
[ "$((_n - _l))" -lt "$WAN_WATCH_COOLDOWN_S" ] 2>/dev/null
}
# ---------------------------------------------------------------------------
# Probes: tunnels, services, kernel — cheap, every run
# ---------------------------------------------------------------------------
probe_emit() {
_ts="$1"; _role="$2"; _host="$3"; _section="$4"
shift 4
emit probe "$_ts" "$_role" "$_host" "section=$_section $*"
}
probe_all() {
_ts="$1"; _role="$2"; _host="$3"
# Kernel modules (GRE/PPTP)
if [ -r /proc/modules ]; then
_gre="$(grep -E '^ip_gre |^gre ' /proc/modules 2>/dev/null | awk '{print $1}' | tr '\n' ',' | sed 's/,$//')"
_pptp="$(grep -E 'nf_conntrack_pptp|nf_nat_pptp' /proc/modules 2>/dev/null | awk '{print $1}' | tr '\n' ',' | sed 's/,$//')"
[ -n "$_gre" ] && probe_emit "$_ts" "$_role" "$_host" gre "loaded=${_gre}"
[ -n "$_pptp" ] && probe_emit "$_ts" "$_role" "$_host" pptp_mod "loaded=${_pptp}"
fi
# GRE / gretap links
ip -d link show type gre 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" '
/^[0-9]+:/{name=$2; gsub(":","",name); state=$0; getline; detail=$0;
printf "kind=probe ts=%s ver=3 role=%s host=%s section=gre_iface name=%s up=%s\n", ts,role,host,name,(index(state,"UP")?"yes":"no")}
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
# PPTP listeners / sessions
_p1723="$(ss -lnpt 2>/dev/null | grep -c ':1723 ' || echo 0)"
_pppd="$(pgrep -c pppd 2>/dev/null || echo 0)"
_pptpd="$(pgrep -c pptpd 2>/dev/null || echo 0)"
probe_emit "$_ts" "$_role" "$_host" pptp "listen1723=${_p1723} pppd=${_pppd} pptpd=${_pptpd}"
# tinc
_tinc="$(pgrep -c tincd 2>/dev/null || echo 0)"
if [ "$_tinc" -gt 0 ] 2>/dev/null; then
_tn=""
if command -v tincctl >/dev/null 2>&1; then
for _net in /etc/tinc/*/tinc.conf; do
[ -f "$_net" ] || continue
_n="$(basename "$(dirname "$_net")")"
_st="$(tincctl -n "$_n" status 2>/dev/null | tr '\n' ' ' | sed 's/ */ /g')"
probe_emit "$_ts" "$_role" "$_host" tinc "net=${_n} tincd=${_tinc} status=${_st:-unknown}"
done
else
probe_emit "$_ts" "$_role" "$_host" tinc "tincd=${_tinc} status=no_tincctl"
fi
else
probe_emit "$_ts" "$_role" "$_host" tinc "tincd=0"
fi
# WireGuard
if command -v wg >/dev/null 2>&1; then
wg show all 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" '
/^interface:/ { iface=$2; peer=""; hs=""; ep=""; tx=""; rx="" }
/^ public key:/ { pk=$3 }
/^ endpoint:/ { ep=$2 }
/^ latest handshake:/ { hs=$0; sub(/^ latest handshake: /,"",hs) }
/^ transfer:/ { tx=$2; rx=$4 }
/^$/ {
if (iface != "")
printf "kind=probe ts=%s ver=3 role=%s host=%s section=wireguard iface=%s endpoint=%s handshake=%s tx=%s rx=%s\n",
ts, role, host, iface, ep, hs, tx, rx
}
END {
if (iface != "")
printf "kind=probe ts=%s ver=3 role=%s host=%s section=wireguard iface=%s endpoint=%s handshake=%s tx=%s rx=%s\n",
ts, role, host, iface, ep, hs, tx, rx
}
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
fi
# iodine / dns tunnel iface
_iod="$(pgrep -c iodined 2>/dev/null || echo 0)"
probe_emit "$_ts" "$_role" "$_host" iodine "iodined=${_iod}"
for _d in dns0 tun0 tap0; do
ip link show "$_d" >/dev/null 2>&1 || continue
ip -s link show "$_d" 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v d="$_d" '
/RX:/{getline; re=$3; rd=$4} /TX:/{getline; te=$3; td=$4}
END{printf "kind=probe ts=%s ver=3 role=%s host=%s section=tunnel_iface dev=%s rx_err=%s rx_drop=%s tx_err=%s tx_drop=%s\n",
ts,role,host,d,re+0,rd+0,te+0,td+0}
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
done
# Named / squid / nginx (edge services)
for _svc in named bind9 squid nginx iodined pptpd; do
_c="$(pgrep -c "$_svc" 2>/dev/null || echo 0)"
[ "$_c" -gt 0 ] 2>/dev/null || continue
_cpu="$(ps -eo pcpu,comm 2>/dev/null | awk -v s="$_svc" '$2~s{print $1; exit}')"
probe_emit "$_ts" "$_role" "$_host" service "name=${_svc} procs=${_c} cpu=${_cpu:-n/a}"
done
# Key ifaces stats
for _d in xenbr0 eno0 eth0 wlan0 br0; do
ip link show "$_d" >/dev/null 2>&1 || continue
ip -s link show "$_d" 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v d="$_d" '
/RX:/{getline; re=$3; rd=$4} /TX:/{getline; te=$3; td=$4}
END{printf "kind=probe ts=%s ver=3 role=%s host=%s section=iface dev=%s rx_drop=%s tx_drop=%s rx_err=%s tx_err=%s\n",
ts,role,host,d,rd+0,td+0,re+0,te+0}
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
tc -s qdisc show dev "$_d" 2>/dev/null | head -3 | while IFS= read -r _ql; do
probe_emit "$_ts" "$_role" "$_host" qdisc "dev=${_d} detail=${_ql}"
done
done
# softnet drops
if [ -r /proc/net/softnet_stat ]; then
awk -v ts="$_ts" -v role="$_role" -v host="$_host" '$2+0>0{
printf "kind=probe ts=%s ver=3 role=%s host=%s section=softnet cpu=%d dropped=%s\n",ts,role,host,NR-1,$2
}' /proc/net/softnet_stat >> "$WAN_WATCH_LOG" 2>/dev/null || true
fi
# Routes + neigh for key targets
for _label in wan:$WAN_PING_TARGET fe:$FE_PING_TARGET gw:${GW_PING_TARGET:-auto}; do
_l="${_label%%:*}"; _t="${_label#*:}"
[ "$_t" = "auto" ] && _t="$(default_gw)"
[ -n "$_t" ] && [ "$_t" != none ] || continue
ip route get "$_t" 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v l="$_l" -v t="$_t" '
{dev=via=src=""; for(i=1;i<=NF;i++){if($i=="dev")dev=$(i+1);if($i=="via")via=$(i+1);if($i=="src")src=$(i+1)}
printf "kind=probe ts=%s ver=3 role=%s host=%s section=route label=%s target=%s dev=%s via=%s src=%s\n",ts,role,host,l,t,dev,via,src}
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
done
_ns="$(neigh_state "$FE_PING_TARGET")"
[ -n "$_ns" ] && probe_emit "$_ts" "$_role" "$_host" neigh "ip=${FE_PING_TARGET} state=${_ns}"
# iptables hot rules (router/fe)
if command -v iptables >/dev/null 2>&1 && [ "$_role" = "router" ]; then
for _spec in filter:FORWARD nat:PREROUTING nat:POSTROUTING; do
_tbl="${_spec%%:*}"; _ch="${_spec#*:}"
iptables -t "$_tbl" -L "$_ch" -n -v --line-numbers 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v tbl="$_tbl" -v ch="$_ch" '
/^[0-9]+/ && $2+0>1000 {
printf "kind=probe ts=%s ver=3 role=%s host=%s section=iptables tbl=%s chain=%s line=%s pkts=%s target=%s\n",
ts,role,host,tbl,ch,$1,$2,$4
}' | head -6 >> "$WAN_WATCH_LOG" 2>/dev/null || true
done
fi
# Top CPU
ps -eo pcpu=,pmem=,comm= --sort=-pcpu 2>/dev/null | head -5 | awk -v ts="$_ts" -v role="$_role" -v host="$_host" '
NF>=3{printf "kind=probe ts=%s ver=3 role=%s host=%s section=proc rank=%d cpu=%s mem=%s comm=%s\n",ts,role,host,NR,$1,$2,$3}
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
ss -s 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" '/TCP:|UDP:/{
gsub(/^[ \t]+/,""); printf "kind=probe ts=%s ver=3 role=%s host=%s section=ss %s\n",ts,role,host,$0
}' >> "$WAN_WATCH_LOG" 2>/dev/null || true
}
# ---------------------------------------------------------------------------
# Deep capture (on anomaly)
# ---------------------------------------------------------------------------
deep_burst_line() {
_ts="$1"; _role="$2"; _host="$3"; _label="$4"; _target="$5"
set -- $(ping_stats "$_target" "$DEEP_PING_COUNT" "$DEEP_PING_DEADLINE_S")
emit deep "$_ts" "$_role" "$_host" \
"parent_ts=$_ts section=burst label=$_label target=$_target loss=$1% min_ms=$2 avg_ms=$3 max_ms=$4 mdev_ms=$5"
}
deep_capture() {
_ts="$1"; _role="$2"; _host="$3"; _triggers="$4"
emit alert "$_ts" "$_role" "$_host" "parent_ts=$_ts triggers=\"$_triggers\""
_ct="$(conntrack_kv)"
emit deep "$_ts" "$_role" "$_host" "parent_ts=$_ts section=host load1=$(load_1m) cpu_busy=$(cpu_busy_pct)% mem_avail_kb=$(awk '/^MemAvailable:/{print $2}' /proc/meminfo 2>/dev/null || echo n/a) ${_ct}"
deep_burst_line "$_ts" "$_role" "$_host" wan "$WAN_PING_TARGET"
deep_burst_line "$_ts" "$_role" "$_host" fe "$FE_PING_TARGET"
_gw="$GW_PING_TARGET"; [ "$_gw" = auto ] && _gw="$(default_gw)"
[ -n "$_gw" ] && [ "$_gw" != none ] && deep_burst_line "$_ts" "$_role" "$_host" gw "$_gw"
probe_all "$_ts" "$_role" "$_host"
}
detect_triggers() {
_line="$1"; _role="$2"; _t=""
for _p in wan fe be gw router; do
_a="$(field_val "${_p}_avg_ms" "$_line")"
_l="$(field_val "${_p}_loss" "$_line")"
_m="$(field_val "${_p}_mdev_ms" "$_line")"
_thr="$FE_ALERT_MS"
case "$_p" in wan) _thr="$WAN_ALERT_MS";; fe) _thr="$FE_ALERT_MS";;
be) _thr="$BE_ALERT_MS";; gw) _thr="$GW_ALERT_MS";;
router) _thr="$ROUTER_ALERT_MS";; esac
num_ge "$_a" "$_thr" && _t="${_t}${_t:+,}${_p}_avg=${_a}"
num_ge "$_l" "$LOSS_ALERT_PCT" && _t="${_t}${_t:+,}${_p}_loss=${_l}"
num_ge "$_m" "$MDEV_ALERT_MS" && _t="${_t}${_t:+,}${_p}_mdev=${_m}"
done
_ct="$(printf '%s\n' "$_line" | sed -n 's/.*(\([0-9]*\)%).*/\1/p' | head -1)"
num_ge "$_ct" "$CONNTRACK_ALERT_PCT" && _t="${_t}${_t:+,}conntrack_pct=${_ct}"
_wan="$(field_val wan_avg_ms "$_line")"; _fe="$(field_val fe_avg_ms "$_line")"
if [ "$_role" = pi ] && num_ge "$_fe" "$FE_ALERT_MS" && ! num_ge "$_wan" "$WAN_ALERT_MS"; then
case "$_t" in *fe_avg=*) ;; *) _t="${_t}${_t:+,}pi_fe_path";; esac
fi
if [ "$_role" = router ] && num_ge "$_wan" "$WAN_ALERT_MS" && ! num_ge "$_fe" "$FE_ALERT_MS"; then
case "$_t" in *wan_avg=*) ;; *) _t="${_t}${_t:+,}router_wan_path";; esac
fi
printf '%s' "$_t"
}
# ---------------------------------------------------------------------------
# Verdict engine — runs EVERY time on current snapshot
# ---------------------------------------------------------------------------
compute_verdict() {
_line="$1"; _role="$2"
_probe_hint="${3:-}"
_wan="$(field_val wan_avg_ms "$_line")"
_fe="$(field_val fe_avg_ms "$_line")"
_be="$(field_val be_avg_ms "$_line")"
_gw="$(field_val gw_avg_ms "$_line")"
_wmdev="$(field_val wan_mdev_ms "$_line")"
_ct="$(printf '%s\n' "$_line" | sed -n 's/.*(\([0-9]*\)%).*/\1/p' | head -1)"
_verdict="OK"
_conf="high"
_summary="All probes within thresholds."
_action="none"
if [ "$_role" = router ]; then
if num_ge "$_wan" "$WAN_ALERT_MS" && ! num_ge "$_fe" 5; then
_verdict="MONSTRO_WAN_BLOAT"
_summary="WAN leg slow (~${_wan}ms) but FE local OK (~${_fe}ms). Edge xenbr0/upstream queue."
_action="SQM/fq_codel on xenbr0; check gw $(route_via "$WAN_PING_TARGET" 2>/dev/null || echo 10.7.16.228); review squid/named load"
elif num_ge "$_gw" "$GW_ALERT_MS" && num_ge "$_wan" "$WAN_ALERT_MS"; then
_verdict="UPSTREAM_GW"
_summary="WAN (${_wan}ms) and gw (${_gw}ms) both high — parent hop 10.7.16.228 stressed."
_action="Check upstream link; correlate with ISP/gateway admin"
elif num_ge "$_ct" "$CONNTRACK_ALERT_PCT"; then
_verdict="CONNTRACK_PRESSURE"
_summary="Conntrack at ${_ct}% — new flows may stall."
_action="Inspect conntrack count; reduce NAT/tunnel sessions or raise nf_conntrack_max"
elif num_ge "$_wmdev" "$MDEV_ALERT_MS"; then
_verdict="WAN_JITTER"
_summary="WAN jitter high (mdev=${_wmdev}ms) avg=${_wan}ms."
_action="Bufferbloat likely; enable fq_codel"
fi
elif [ "$_role" = pi ]; then
if num_ge "$_fe" "$FE_ALERT_MS" && ! num_ge "$_wan" "$WAN_ALERT_MS"; then
_verdict="PATH_TO_FE"
_summary="Pi ISP OK (~${_wan}ms) but FE path slow (~${_fe}ms). Problem at monstro edge, not Pi WiFi."
_action="Check monstro wan-watch verdict; your traffic crosses modem WAN queue"
elif num_ge "$_wan" "$WAN_ALERT_MS"; then
_verdict="PI_ISP"
_summary="Pi WAN to 1.1.1.1 slow (${_wan}ms)."
_action="Local ISP/WiFi issue on Pi side"
fi
elif [ "$_role" = fe ]; then
if num_ge "$_wan" "$WAN_ALERT_MS"; then
_verdict="FE_WAN_EGRESS"
_summary="FE WAN egress slow (${_wan}ms)."
_action="Check FE default route and nginx upstream separately"
fi
fi
# Probe hints from same run (grep log tail — set by caller in _probe_hint)
case "${_probe_hint:-}" in
*dns0*tx_drop=*|*tunnel_iface*dns0*)
if [ "$_verdict" = "OK" ] || [ "$_verdict" = "MONSTRO_WAN_BLOAT" ]; then
_verdict="TUNNEL_DNS0_LOAD"
_summary="${_summary} iodine/dns0 tunnel drops elevated."
_action="${_action}; inspect iodined and dns0 tx_drop"
_conf="medium"
fi ;;
*named*cpu=*)
if [ "$_verdict" = "MONSTRO_WAN_BLOAT" ] || [ "$_verdict" = "OK" ]; then
_summary="${_summary} named DNS active on router."
_conf="medium"
fi ;;
esac
printf 'verdict=%s confidence=%s summary="%s" action="%s" wan_ms=%s fe_ms=%s gw_ms=%s' \
"$_verdict" "$_conf" "$_summary" "$_action" "$_wan" "$_fe" "$_gw"
}
# ---------------------------------------------------------------------------
# Main cron run
# ---------------------------------------------------------------------------
run_watch() {
command -v ping >/dev/null 2>&1 || { echo "wan-watch: need ping" >&2; exit 1; }
acquire_lock
_ts="$(date -Iseconds 2>/dev/null || date '+%Y-%m-%dT%H:%M:%S%z')"
_role="$(detect_role)"
_host="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)"
_gw="$GW_PING_TARGET"; [ "$_gw" = auto ] && _gw="$(default_gw)"
_ct="$(conntrack_kv)"
_wdev="$(route_dev "$WAN_PING_TARGET")"; _fdev="$(route_dev "$FE_PING_TARGET")"
_wgw="$(route_via "$WAN_PING_TARGET")"
_ns="$(neigh_state "$FE_PING_TARGET")"
_snap="$(append_ping "" wan "$WAN_PING_TARGET")"
_snap="$(append_ping "$_snap" fe "$FE_PING_TARGET")"
[ -n "$BE_PING_TARGET" ] && [ "$BE_PING_TARGET" != none ] && _snap="$(append_ping "$_snap" be "$BE_PING_TARGET")"
[ -n "$ROUTER_PING_TARGET" ] && [ "$ROUTER_PING_TARGET" != none ] && _snap="$(append_ping "$_snap" router "$ROUTER_PING_TARGET")"
[ -n "$_gw" ] && [ "$_gw" != none ] && _snap="$(append_ping "$_snap" gw "$_gw")"
_snap="$_snap load1=$(load_1m) cpu_busy=$(cpu_busy_pct)%"
[ -n "$_ct" ] && _snap="$_snap $_ct"
[ -n "$_wdev" ] && _snap="$_snap route_wan=$_wdev"
[ -n "$_fdev" ] && _snap="$_snap route_fe=$_fdev"
[ -n "$_wgw" ] && _snap="$_snap via_wan=$_wgw"
[ -n "$_ns" ] && _snap="$_snap neigh_fe=$_ns"
emit snapshot "$_ts" "$_role" "$_host" "$_snap" >/dev/null
probe_all "$_ts" "$_role" "$_host"
_probe_hint="$(tail -30 "$WAN_WATCH_LOG" 2>/dev/null | grep 'kind=probe' | tr '\n' ' ')"
_verdict_kv="$(compute_verdict "$_snap" "$_role" "$_probe_hint")"
_verdict_line="kind=verdict ts=$_ts ver=3 role=$_role host=$_host $_verdict_kv"
printf '%s\n' "$_verdict_line" >> "$WAN_WATCH_LOG"
_triggers="$(detect_triggers "$_snap" "$_role")"
_do_deep=0
if [ -n "$_triggers" ]; then _do_deep=1; fi
if [ "$WAN_WATCH_DEEP" = "always" ]; then _do_deep=1; fi
if [ "$WAN_WATCH_DEEP" = "never" ]; then _do_deep=0; fi
if [ "$_do_deep" = 1 ]; then
if cooldown_active && [ "$WAN_WATCH_DEEP" != "always" ]; then
emit alert "$_ts" "$_role" "$_host" "parent_ts=$_ts triggers=\"$_triggers\" action=skipped_cooldown" >/dev/null
else
deep_capture "$_ts" "$_role" "$_host" "$_triggers"
date +%s > "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null || true
fi
fi
# Human one-liner to cron log
printf 'WAN-WATCH [%s/%s] %s\n' "$_role" "$_host" "$_verdict_kv"
}
# ---------------------------------------------------------------------------
# --analyze: final conclusion from log without waiting
# ---------------------------------------------------------------------------
analyze_log() {
_log="$WAN_WATCH_LOG"
_hours="${1:-$WAN_ANALYZE_HOURS}"
[ -r "$_log" ] || { echo "wan-watch: no log $_log" >&2; exit 1; }
_cut="$(date -d "${_hours} hours ago" '+%Y-%m-%dT%H:%M' 2>/dev/null || echo "")"
echo "=== wan-watch analyze (last ${_hours}h) ==="
echo "log=$_log"
echo ""
awk -v cut="$_cut" '
function ts_ok(t) { return (cut == "" || t >= cut) }
/^kind=snapshot / && ts_ok($0) {
for(i=1;i<=NF;i++) {
if($i~/^wan_avg_ms=/){split($i,a,"="); wa=a[2]+0; wn++}
if($i~/^fe_avg_ms=/){split($i,a,"="); fa=a[2]+0; fn++}
if($i~/^gw_avg_ms=/){split($i,a,"="); ga=a[2]+0; gn++}
if($i~/^role=/){split($i,a,"="); role=a[2]}
}
if(wa>=50){wb++} else if(wa>=0){wg++}
if(fa>=50){fb++} else if(fa>=0){fg++}
if(wa>=50 && fa<5) rw_ok_fe++ # router pattern
if(fa>=50 && wa<30) pi_bad++ # pi pattern
sn++
}
/^kind=verdict / && ts_ok($0) {
if($0~/verdict=OK/) okv++; else badv++
for(i=1;i<=NF;i++) if($i~/^verdict=/){split($i,a,"="); vc[a[2]]++}
}
/^kind=probe / && /section=tunnel_iface/ && /tx_drop=[1-9]/ { dns_drop++ }
/^kind=probe / && /section=service/ && /name=named/ { named++ }
/^kind=probe / && /section=service/ && /name=squid/ { squid++ }
END {
printf "snapshots=%d (wan_bad=%d wan_good=%d fe_bad=%d fe_good=%d)\n", sn, wb+0, wg+0, fb+0, fg+0
printf "router_wan_bad_fe_local_ok=%d (MONSTRO_WAN_BLOAT signature)\n", rw_ok_fe+0
printf "pi_fe_bad_wan_ok=%d (PATH_TO_FE signature)\n", pi_bad+0
printf "verdicts: ok=%d anomaly=%d\n", okv+0, badv+0
printf "verdict_counts:"
for(v in vc) printf " %s=%d", v, vc[v]
printf "\n"
printf "probe_hints: dns0_tx_drop_events=%d named_samples=%d squid_samples=%d\n", dns_drop+0, named+0, squid+0
pct = (sn>0) ? int(100*(wb+fb)/(2*sn)) : 0
printf "\n"
if(rw_ok_fe>=3 || pi_bad>=3) {
printf "CONCLUSION: MONSTRO WAN EDGE QUEUEING (confidence=HIGH)\n"
printf " Evidence: wan slow + fe local OK; Pi ISP stays ~9ms in prior data.\n"
printf " Not: FE VM, BE, Pi WiFi, global ISP.\n"
printf " Fix order: 1) fq_codel/SQM on xenbr0 2) gw 10.7.16.228 3) squid/named 4) iodine dns0\n"
} else if(wb+fb==0) {
printf "CONCLUSION: HEALTHY (confidence=HIGH)\n"
} else {
printf "CONCLUSION: MIXED/INTERMITTENT (%d%% bad samples) — keep */5 cron running\n", pct
}
}
' "$_log"
echo ""
echo "=== last 5 verdicts ==="
grep '^kind=verdict ' "$_log" 2>/dev/null | tail -5 | while IFS= read -r _l; do
_t="$(printf '%s\n' "$_l" | sed 's/.*ts=\([^ ]*\).*/\1/')"
_v="$(printf '%s\n' "$_l" | sed 's/.*verdict=\([^ ]*\).*/\1/')"
_s="$(printf '%s\n' "$_l" | sed 's/.*summary="\([^"]*\)".*/\1/')"
printf ' %s %s %s\n' "$_t" "$_v" "$_s"
done
}
timeline_log() {
_log="$WAN_WATCH_LOG"
[ -r "$_log" ] || { echo "no log"; exit 1; }
echo "=== hourly (snapshot wan/fe bad if avg>=50) ==="
awk '
/^kind=snapshot / {
ts=$0; sub(/.*ts=/,"",ts); sub(/ .*/,"",ts)
hour=substr(ts,1,13)
for(i=1;i<=NF;i++){
if($i~/^wan_avg_ms=/){split($i,a,"="); w=a[2]+0}
if($i~/^fe_avg_ms=/){split($i,a,"="); f=a[2]+0}
}
t[hour]++; if(w>=50||f>=50) b[hour]++
}
END {
for(h in t){ bh=b[h]+0; printf " %s %d/%d bad ", h, bh, t[h]
for(i=0;i<t[h];i++) printf(i<bh?"#":".")
printf "\n"
}
}
' "$_log" | sort
}
full_report() {
_ts="$(date -Iseconds 2>/dev/null || date '+%Y-%m-%dT%H:%M:%S%z')"
_role="$(detect_role)"
_host="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)"
echo "=== wan-watch FULL @ $_ts role=$_role host=$_host ==="
PING_COUNT=5 PING_DEADLINE_S=10
for _t in "$WAN_PING_TARGET" "$FE_PING_TARGET" "$BE_PING_TARGET" "$ROUTER_PING_TARGET" "$(default_gw)"; do
[ -n "$_t" ] && [ "$_t" != none ] || continue
echo "--- ping $_t ---"
ping -c 5 -w 10 "$_t" 2>/dev/null || echo "fail"
done
echo "--- host ---"
echo "load=$(load_1m) cpu=$(cpu_busy_pct)% $(conntrack_kv)"
echo "--- probes ---"
probe_all "$_ts" "$_role" "$_host"
tail -40 "$WAN_WATCH_LOG" | grep "^kind=probe "
echo "--- verdict ---"
_snap="ts=$_ts role=$_role host=$_host"
_snap="$(append_ping "$_snap" wan "$WAN_PING_TARGET")"
_snap="$(append_ping "$_snap" fe "$FE_PING_TARGET")"
compute_verdict "$_snap" "$_role"
echo ""
}
correlate_logs() {
if [ "$#" -lt 1 ]; then echo "wan-watch: --correlate needs log files" >&2; exit 1; fi
_tmp="$(mktemp "${TMPDIR:-/tmp}/wan-corr.XXXXXX")"
trap 'rm -f "$_tmp"' EXIT INT HUP
for _path in "$@"; do
[ -r "$_path" ] || continue
while IFS= read -r _line || [ -n "$_line" ]; do
case "$_line" in
kind=snapshot*) _ts="${_line#*ts=}"; _ts="${_ts%% *}" ;;
ts=*) _ts="${_line#ts=}"; _ts="${_ts%% *}" ;;
*) continue ;;
esac
_date="${_ts%%T*}"; _rest="${_ts#*T}"; _hour="${_rest%%:*}"
_min="${_rest#*:}"; _min="${_min%%:*}"; _min="${_min%%+*}"; _min="${_min%%-*}"
_slot="${_date}T${_hour}:$(printf '%02d' $((_min / 15 * 15)))"
_role="$(printf '%s\n' "$_line" | awk '{for(i=1;i<=NF;i++) if($i~/^role=/){split($i,a,"=");print a[2];exit}}')"
[ -n "$_role" ] || _role="$(basename "$_path" .log)"
printf '%s\t%s\t%s\t%s\n' "$_slot" "$_role" "$_ts" "$_line" >> "$_tmp"
done < "$_path"
done
printf 'slot\t'
_roles="$(awk -F'\t' '{print $2}' "$_tmp" | sort -u | tr '\n' ' ')"
for _r in $_roles; do printf '%s_wan\t%s_fe\t%s_verdict\t' "$_r" "$_r" "$_r"; done
printf '\n'
awk -F'\t' '
{
slot=$1; role=$2; ts=$3; line=$4; key=slot SUBSEP role
if (!(key in best) || ts>best[key]) {
best[key]=ts; wan=fe="-"; ver="-"
for(i=1;i<=split(line,a," ");i++){
if(a[i]~/^wan_avg_ms=/){split(a[i],b,"=");wan=b[2]}
if(a[i]~/^fe_avg_ms=/){split(a[i],b,"=");fe=b[2]}
}
W[slot,role,"wan"]=wan; W[slot,role,"fe"]=fe
slots[slot]=1; roles[role]=1
}
}
END{
nr=0; for(r in roles) rl[++nr]=r; asort(rl)
ns=0; for(s in slots) sl[++ns]=s; asort(sl)
for(i=1;i<=ns;i++){
s=sl[i]; printf "%s", s
for(j=1;j<=nr;j++){
r=rl[j]; printf "\t%s\t%s\t-", W[s,r,"wan"]+0?W[s,r,"wan"]:"-", W[s,r,"fe"]+0?W[s,r,"fe"]:"-"
}
printf "\n"
}
}
' "$_tmp"
}
usage() {
cat <<EOF
wan-watch.sh — unified network watchdog (ver=3)
Cron: */5 * * * * root . /etc/conf.d/wan-watch; /usr/local/bin/wan-watch.sh
Commands:
(default) snapshot + probes + verdict (+ deep if bad)
--full immediate human report
--analyze [H] conclusion from log (default ${WAN_ANALYZE_HOURS}h)
--timeline hourly bad/good bars
--force-deep set WAN_WATCH_DEEP=always for one run
--tail [N] last N snapshot+verdict lines
--correlate FILE… merge Pi+router logs by 15-min slot
--help
Log: $WAN_WATCH_LOG (kind=snapshot|probe|verdict|alert|deep)
EOF
}
case "${1:-}" in
-h|--help|help) usage ;;
--full|full) full_report ;;
--analyze|analyze) analyze_log "${2:-$WAN_ANALYZE_HOURS}" ;;
--timeline|timeline) timeline_log ;;
--force-deep|force-deep) WAN_WATCH_DEEP=always run_watch ;;
--tail|tail)
_n="${2:-20}"
grep -E '^kind=(snapshot|verdict|alert) ' "$WAN_WATCH_LOG" 2>/dev/null | tail -n "$_n" || true
;;
"") run_watch ;;
*) echo "wan-watch: unknown $1"; usage; exit 1 ;;
esac
Reference in New Issue View Git Blame Copy Permalink