# wan-deep-diag.sh — parseable deep diagnostics (sourced by wan-snapshot.sh). # Not meant to be executed directly. # # Emits kind=deep lines to WAN_WATCH_LOG (key=value, one fact per line). wan_deep_load_1m() { awk '{print $1}' /proc/loadavg 2>/dev/null || echo "n/a" } wan_deep_cpu_busy_pct() { awk '/^cpu / { idle = $5 + $6; total = 0 for (i = 2; i <= NF; i++) total += $i if (total > 0) printf "%.0f", 100 * (total - idle) / total }' /proc/stat 2>/dev/null || echo "n/a" } wan_deep_mem_avail_kb() { awk '/^MemAvailable:/ { print $2 }' /proc/meminfo 2>/dev/null || echo "n/a" } wan_deep_conntrack_fields() { _cur="$(cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null)" || _cur="" _max="$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null)" || _max="" if [ -n "$_cur" ] && [ -n "$_max" ] && [ "$_max" -gt 0 ] 2>/dev/null; then _pct=$((_cur * 100 / _max)) printf 'conntrack_cur=%s conntrack_max=%s conntrack_pct=%s' "$_cur" "$_max" "$_pct" else printf 'conntrack_cur=n/a conntrack_max=n/a conntrack_pct=n/a' fi } wan_deep_emit() { # wan_deep_emit KIND TS ROLE HOST [key=value ...] _kind="$1" _ts="$2" _role="$3" _host="$4" shift 4 _line="kind=$_kind ts=$_ts role=$_role host=$_host $*" _dir="$(dirname "$WAN_WATCH_LOG")" [ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true printf '%s\n' "$_line" >> "$WAN_WATCH_LOG" } # Burst ping (fast) -> one kind=deep line. wan_deep_burst_ping() { _parent_ts="$1" _role="$2" _hostn="$3" _label="$4" _target="$5" _count="${6:-5}" _deadline="${7:-6}" _out="$(ping -c "$_count" -w "$_deadline" "$_target" 2>/dev/null)" || _out="" if [ -z "$_out" ]; then wan_deep_emit deep "$_parent_ts" "$_role" "$_hostn" \ "parent_ts=$_parent_ts section=burst label=$_label target=$_target loss=-1 avg_ms=-1" return 0 fi _loss="$(printf '%s\n' "$_out" | awk '/packet loss/ { for (i = 1; i <= NF; i++) if ($i ~ /%$/) { gsub(/%/, "", $i); print $i; exit } }')" _rtt="$(printf '%s\n' "$_out" | sed -n 's/.*= \([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\) ms.*/\1 \2 \3 \4/p' | head -1)" _min="-1" _avg="-1" _max="-1" _mdev="-1" if [ -n "$_rtt" ]; then _min="${_rtt%% *}" _rest="${_rtt#* }" _avg="${_rest%% *}" _rest="${_rest#* }" _max="${_rest%% *}" _mdev="${_rest#* }" fi [ -n "$_loss" ] || _loss="-1" wan_deep_emit deep "$_parent_ts" "$_role" "$_hostn" \ "parent_ts=$_parent_ts section=burst label=$_label target=$_target loss=${_loss}% min_ms=$_min avg_ms=$_avg max_ms=$_max mdev_ms=$_mdev" } wan_deep_host_health() { _parent_ts="$1" _role="$2" _hostn="$3" _ct="$(wan_deep_conntrack_fields)" wan_deep_emit deep "$_parent_ts" "$_role" "$_hostn" \ "parent_ts=$_parent_ts section=host load1=$(wan_deep_load_1m) cpu_busy=$(wan_deep_cpu_busy_pct)% mem_avail_kb=$(wan_deep_mem_avail_kb) $_ct" } wan_deep_top_procs() { _parent_ts="$1" _role="$2" _hostn="$3" _n="${4:-5}" ps -eo pcpu=,pmem=,comm= --sort=-pcpu 2>/dev/null | head -n "$_n" | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v n="$_n" ' NF >= 3 { rank = NR printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=proc rank=%d cpu=%s mem=%s comm=%s\n", ts, role, host, ts, rank, $1, $2, $3 } ' >> "$WAN_WATCH_LOG" 2>/dev/null || true } wan_deep_iface_stats() { _parent_ts="$1" _role="$2" _hostn="$3" _dev="$4" [ -n "$_dev" ] || return 0 ip -s link show "$_dev" 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v dev="$_dev" ' /^[0-9]+:/ { rx_b = tx_b = rx_p = tx_p = rx_drop = tx_drop = rx_err = tx_err = 0 } /RX: bytes/ { rx_b = $3; rx_p = $2 } /RX:.*packets/ { getline; rx_err = $3; rx_drop = $4 } /TX: bytes/ { tx_b = $3; tx_p = $2 } /TX:.*packets/ { getline; tx_err = $3; tx_drop = $4 } END { printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=iface dev=%s rx_bytes=%s tx_bytes=%s rx_drop=%s tx_drop=%s rx_err=%s tx_err=%s\n", ts, role, host, ts, dev, rx_b, tx_b, rx_drop, tx_drop, rx_err, tx_err } ' >> "$WAN_WATCH_LOG" 2>/dev/null || true } wan_deep_all_ifaces_nonzero() { _parent_ts="$1" _role="$2" _hostn="$3" ip -s link 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" ' /^[0-9]+:/ { if (iface != "" && iface !~ /^lo$/ && (rx_drop + tx_drop + rx_err + tx_err) > 0) printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=iface dev=%s rx_drop=%s tx_drop=%s rx_err=%s tx_err=%s\n", ts, role, host, ts, iface, rx_drop, tx_drop, rx_err, tx_err iface = $2; gsub(":", "", iface) rx_drop = tx_drop = rx_err = tx_err = 0 } /RX:/ { getline; rx_err = $3; rx_drop = $4 } /TX:/ { getline; tx_err = $3; tx_drop = $4 } END { if (iface != "" && iface !~ /^lo$/ && (rx_drop + tx_drop + rx_err + tx_err) > 0) printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=iface dev=%s rx_drop=%s tx_drop=%s rx_err=%s tx_err=%s\n", ts, role, host, ts, iface, rx_drop, tx_drop, rx_err, tx_err } ' >> "$WAN_WATCH_LOG" 2>/dev/null || true } wan_deep_softnet() { _parent_ts="$1" _role="$2" _hostn="$3" [ -r /proc/net/softnet_stat ] || return 0 awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" ' $2 + 0 > 0 { printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=softnet cpu=%d dropped=%s processed=%s\n", ts, role, host, ts, NR - 1, $2, $1 } ' /proc/net/softnet_stat >> "$WAN_WATCH_LOG" 2>/dev/null || true } wan_deep_neigh() { _parent_ts="$1" _role="$2" _hostn="$3" _ip="$4" [ -n "$_ip" ] || return 0 ip neigh show "$_ip" 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v ip="$_ip" ' NF >= 5 { printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=neigh ip=%s lladdr=%s dev=%s state=%s\n", ts, role, host, ts, ip, $5, $3, $6 } ' >> "$WAN_WATCH_LOG" 2>/dev/null || true } wan_deep_route() { _parent_ts="$1" _role="$2" _hostn="$3" _label="$4" _target="$5" [ -n "$_target" ] || return 0 ip route get "$_target" 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v label="$_label" -v target="$_target" ' { dev = via = ""; src = "" for (i = 1; i <= NF; i++) { if ($i == "dev") dev = $(i + 1) if ($i == "via") via = $(i + 1) if ($i == "src") src = $(i + 1) } printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=route label=%s target=%s dev=%s via=%s src=%s\n", ts, role, host, ts, label, target, dev, via, src } ' >> "$WAN_WATCH_LOG" 2>/dev/null || true } wan_deep_iptables_hot() { _parent_ts="$1" _role="$2" _hostn="$3" command -v iptables >/dev/null 2>&1 || return 0 for _spec in "filter:FORWARD" "nat:PREROUTING" "nat:POSTROUTING"; do _tbl="${_spec%%:*}" _chain="${_spec#*:}" iptables -t "$_tbl" -L "$_chain" -n -v --line-numbers 2>/dev/null \ | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v tbl="$_tbl" -v chain="$_chain" ' /^[0-9]+/ && $2 + 0 > 0 { ln = $1; pkts = $2; bytes = $3; target = $4 printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=iptables tbl=%s chain=%s line=%s pkts=%s bytes=%s target=%s\n", ts, role, host, ts, tbl, chain, ln, pkts, bytes, target } ' | head -n 8 >> "$WAN_WATCH_LOG" 2>/dev/null || true done } wan_deep_qdisc() { _parent_ts="$1" _role="$2" _hostn="$3" _dev="$4" [ -n "$_dev" ] || return 0 tc -s qdisc show dev "$_dev" 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v dev="$_dev" ' /qdisc/ { printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=qdisc dev=%s detail=%s\n", ts, role, host, ts, dev, $0 } /backlog/ { printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=qdisc dev=%s backlog=%s\n", ts, role, host, ts, dev, $0 } ' >> "$WAN_WATCH_LOG" 2>/dev/null || true } wan_deep_ss_summary() { _parent_ts="$1" _role="$2" _hostn="$3" ss -s 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" ' /TCP:|UDP:/ { gsub(/^[ \t]+/, "") printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=ss %s\n", ts, role, host, ts, $0 } ' >> "$WAN_WATCH_LOG" 2>/dev/null || true } # Full deep capture for one anomaly event. wan_deep_diag_run() { _parent_ts="$1" _role="$2" _hostn="$3" _triggers="$4" _wan_target="$5" _fe_target="$6" _gw_target="$7" _route_wan_dev="$8" _route_fe_dev="$9" wan_deep_emit alert "$_parent_ts" "$_role" "$_hostn" \ "parent_ts=$_parent_ts triggers=\"$_triggers\"" wan_deep_host_health "$_parent_ts" "$_role" "$_hostn" wan_deep_burst_ping "$_parent_ts" "$_role" "$_hostn" "wan" "$_wan_target" wan_deep_burst_ping "$_parent_ts" "$_role" "$_hostn" "fe" "$_fe_target" if [ -n "$_gw_target" ] && [ "$_gw_target" != "none" ]; then wan_deep_burst_ping "$_parent_ts" "$_role" "$_hostn" "gw" "$_gw_target" fi wan_deep_route "$_parent_ts" "$_role" "$_hostn" "wan" "$_wan_target" wan_deep_route "$_parent_ts" "$_role" "$_hostn" "fe" "$_fe_target" wan_deep_neigh "$_parent_ts" "$_role" "$_hostn" "$_fe_target" wan_deep_top_procs "$_parent_ts" "$_role" "$_hostn" 5 wan_deep_softnet "$_parent_ts" "$_role" "$_hostn" wan_deep_ss_summary "$_parent_ts" "$_role" "$_hostn" wan_deep_iface_stats "$_parent_ts" "$_role" "$_hostn" "$_route_wan_dev" wan_deep_iface_stats "$_parent_ts" "$_role" "$_hostn" "$_route_fe_dev" wan_deep_all_ifaces_nonzero "$_parent_ts" "$_role" "$_hostn" if [ "$_role" = "router" ]; then wan_deep_iptables_hot "$_parent_ts" "$_role" "$_hostn" wan_deep_qdisc "$_parent_ts" "$_role" "$_hostn" "$_route_wan_dev" fi if [ "$_role" = "pi" ]; then wan_deep_qdisc "$_parent_ts" "$_role" "$_hostn" "$_route_fe_dev" fi } # Loaded marker for wan-snapshot.sh WAN_DEEP_DIAG_LOADED=1