mirror of
git://f0xx.org/android_cast
synced 2026-07-29 09:38:08 +03:00
9acbdbc433
Net topology: sirene (P2P), spider (MCU), spider net (SFU), pautina (net, mesh), hydra (hybrid), code names
287 lines
9.8 KiB
Bash
287 lines
9.8 KiB
Bash
# wan-deep-diag.sh — parseable deep diagnostics (sourced by wan-snapshot.sh).
|
|
# Not meant to be executed directly.
|
|
#
|
|
# Emits kind=deep lines to WAN_WATCH_LOG (key=value, one fact per line).
|
|
|
|
wan_deep_load_1m() {
|
|
awk '{print $1}' /proc/loadavg 2>/dev/null || echo "n/a"
|
|
}
|
|
|
|
wan_deep_cpu_busy_pct() {
|
|
awk '/^cpu / {
|
|
idle = $5 + $6; total = 0
|
|
for (i = 2; i <= NF; i++) total += $i
|
|
if (total > 0) printf "%.0f", 100 * (total - idle) / total
|
|
}' /proc/stat 2>/dev/null || echo "n/a"
|
|
}
|
|
|
|
wan_deep_mem_avail_kb() {
|
|
awk '/^MemAvailable:/ { print $2 }' /proc/meminfo 2>/dev/null || echo "n/a"
|
|
}
|
|
|
|
wan_deep_conntrack_fields() {
|
|
_cur="$(cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null)" || _cur=""
|
|
_max="$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null)" || _max=""
|
|
if [ -n "$_cur" ] && [ -n "$_max" ] && [ "$_max" -gt 0 ] 2>/dev/null; then
|
|
_pct=$((_cur * 100 / _max))
|
|
printf 'conntrack_cur=%s conntrack_max=%s conntrack_pct=%s' "$_cur" "$_max" "$_pct"
|
|
else
|
|
printf 'conntrack_cur=n/a conntrack_max=n/a conntrack_pct=n/a'
|
|
fi
|
|
}
|
|
|
|
wan_deep_emit() {
|
|
# wan_deep_emit KIND TS ROLE HOST [key=value ...]
|
|
_kind="$1"
|
|
_ts="$2"
|
|
_role="$3"
|
|
_host="$4"
|
|
shift 4
|
|
_line="kind=$_kind ts=$_ts role=$_role host=$_host $*"
|
|
_dir="$(dirname "$WAN_WATCH_LOG")"
|
|
[ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true
|
|
printf '%s\n' "$_line" >> "$WAN_WATCH_LOG"
|
|
}
|
|
|
|
# Burst ping (fast) -> one kind=deep line.
|
|
wan_deep_burst_ping() {
|
|
_parent_ts="$1"
|
|
_role="$2"
|
|
_hostn="$3"
|
|
_label="$4"
|
|
_target="$5"
|
|
_count="${6:-5}"
|
|
_deadline="${7:-6}"
|
|
|
|
_out="$(ping -c "$_count" -w "$_deadline" "$_target" 2>/dev/null)" || _out=""
|
|
if [ -z "$_out" ]; then
|
|
wan_deep_emit deep "$_parent_ts" "$_role" "$_hostn" \
|
|
"parent_ts=$_parent_ts section=burst label=$_label target=$_target loss=-1 avg_ms=-1"
|
|
return 0
|
|
fi
|
|
|
|
_loss="$(printf '%s\n' "$_out" | awk '/packet loss/ {
|
|
for (i = 1; i <= NF; i++) if ($i ~ /%$/) { gsub(/%/, "", $i); print $i; exit }
|
|
}')"
|
|
_rtt="$(printf '%s\n' "$_out" | sed -n 's/.*= \([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\) ms.*/\1 \2 \3 \4/p' | head -1)"
|
|
_min="-1" _avg="-1" _max="-1" _mdev="-1"
|
|
if [ -n "$_rtt" ]; then
|
|
_min="${_rtt%% *}"
|
|
_rest="${_rtt#* }"
|
|
_avg="${_rest%% *}"
|
|
_rest="${_rest#* }"
|
|
_max="${_rest%% *}"
|
|
_mdev="${_rest#* }"
|
|
fi
|
|
[ -n "$_loss" ] || _loss="-1"
|
|
|
|
wan_deep_emit deep "$_parent_ts" "$_role" "$_hostn" \
|
|
"parent_ts=$_parent_ts section=burst label=$_label target=$_target loss=${_loss}% min_ms=$_min avg_ms=$_avg max_ms=$_max mdev_ms=$_mdev"
|
|
}
|
|
|
|
wan_deep_host_health() {
|
|
_parent_ts="$1"
|
|
_role="$2"
|
|
_hostn="$3"
|
|
_ct="$(wan_deep_conntrack_fields)"
|
|
wan_deep_emit deep "$_parent_ts" "$_role" "$_hostn" \
|
|
"parent_ts=$_parent_ts section=host load1=$(wan_deep_load_1m) cpu_busy=$(wan_deep_cpu_busy_pct)% mem_avail_kb=$(wan_deep_mem_avail_kb) $_ct"
|
|
}
|
|
|
|
wan_deep_top_procs() {
|
|
_parent_ts="$1"
|
|
_role="$2"
|
|
_hostn="$3"
|
|
_n="${4:-5}"
|
|
ps -eo pcpu=,pmem=,comm= --sort=-pcpu 2>/dev/null | head -n "$_n" | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v n="$_n" '
|
|
NF >= 3 {
|
|
rank = NR
|
|
printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=proc rank=%d cpu=%s mem=%s comm=%s\n",
|
|
ts, role, host, ts, rank, $1, $2, $3
|
|
}
|
|
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
|
|
}
|
|
|
|
wan_deep_iface_stats() {
|
|
_parent_ts="$1"
|
|
_role="$2"
|
|
_hostn="$3"
|
|
_dev="$4"
|
|
[ -n "$_dev" ] || return 0
|
|
ip -s link show "$_dev" 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v dev="$_dev" '
|
|
/^[0-9]+:/ { rx_b = tx_b = rx_p = tx_p = rx_drop = tx_drop = rx_err = tx_err = 0 }
|
|
/RX: bytes/ { rx_b = $3; rx_p = $2 }
|
|
/RX:.*packets/ { getline; rx_err = $3; rx_drop = $4 }
|
|
/TX: bytes/ { tx_b = $3; tx_p = $2 }
|
|
/TX:.*packets/ { getline; tx_err = $3; tx_drop = $4 }
|
|
END {
|
|
printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=iface dev=%s rx_bytes=%s tx_bytes=%s rx_drop=%s tx_drop=%s rx_err=%s tx_err=%s\n",
|
|
ts, role, host, ts, dev, rx_b, tx_b, rx_drop, tx_drop, rx_err, tx_err
|
|
}
|
|
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
|
|
}
|
|
|
|
wan_deep_all_ifaces_nonzero() {
|
|
_parent_ts="$1"
|
|
_role="$2"
|
|
_hostn="$3"
|
|
ip -s link 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" '
|
|
/^[0-9]+:/ {
|
|
if (iface != "" && iface !~ /^lo$/ && (rx_drop + tx_drop + rx_err + tx_err) > 0)
|
|
printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=iface dev=%s rx_drop=%s tx_drop=%s rx_err=%s tx_err=%s\n",
|
|
ts, role, host, ts, iface, rx_drop, tx_drop, rx_err, tx_err
|
|
iface = $2; gsub(":", "", iface)
|
|
rx_drop = tx_drop = rx_err = tx_err = 0
|
|
}
|
|
/RX:/ { getline; rx_err = $3; rx_drop = $4 }
|
|
/TX:/ { getline; tx_err = $3; tx_drop = $4 }
|
|
END {
|
|
if (iface != "" && iface !~ /^lo$/ && (rx_drop + tx_drop + rx_err + tx_err) > 0)
|
|
printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=iface dev=%s rx_drop=%s tx_drop=%s rx_err=%s tx_err=%s\n",
|
|
ts, role, host, ts, iface, rx_drop, tx_drop, rx_err, tx_err
|
|
}
|
|
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
|
|
}
|
|
|
|
wan_deep_softnet() {
|
|
_parent_ts="$1"
|
|
_role="$2"
|
|
_hostn="$3"
|
|
[ -r /proc/net/softnet_stat ] || return 0
|
|
awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" '
|
|
$2 + 0 > 0 {
|
|
printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=softnet cpu=%d dropped=%s processed=%s\n",
|
|
ts, role, host, ts, NR - 1, $2, $1
|
|
}
|
|
' /proc/net/softnet_stat >> "$WAN_WATCH_LOG" 2>/dev/null || true
|
|
}
|
|
|
|
wan_deep_neigh() {
|
|
_parent_ts="$1"
|
|
_role="$2"
|
|
_hostn="$3"
|
|
_ip="$4"
|
|
[ -n "$_ip" ] || return 0
|
|
ip neigh show "$_ip" 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v ip="$_ip" '
|
|
NF >= 5 {
|
|
printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=neigh ip=%s lladdr=%s dev=%s state=%s\n",
|
|
ts, role, host, ts, ip, $5, $3, $6
|
|
}
|
|
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
|
|
}
|
|
|
|
wan_deep_route() {
|
|
_parent_ts="$1"
|
|
_role="$2"
|
|
_hostn="$3"
|
|
_label="$4"
|
|
_target="$5"
|
|
[ -n "$_target" ] || return 0
|
|
ip route get "$_target" 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v label="$_label" -v target="$_target" '
|
|
{
|
|
dev = via = ""; src = ""
|
|
for (i = 1; i <= NF; i++) {
|
|
if ($i == "dev") dev = $(i + 1)
|
|
if ($i == "via") via = $(i + 1)
|
|
if ($i == "src") src = $(i + 1)
|
|
}
|
|
printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=route label=%s target=%s dev=%s via=%s src=%s\n",
|
|
ts, role, host, ts, label, target, dev, via, src
|
|
}
|
|
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
|
|
}
|
|
|
|
wan_deep_iptables_hot() {
|
|
_parent_ts="$1"
|
|
_role="$2"
|
|
_hostn="$3"
|
|
command -v iptables >/dev/null 2>&1 || return 0
|
|
for _spec in "filter:FORWARD" "nat:PREROUTING" "nat:POSTROUTING"; do
|
|
_tbl="${_spec%%:*}"
|
|
_chain="${_spec#*:}"
|
|
iptables -t "$_tbl" -L "$_chain" -n -v --line-numbers 2>/dev/null \
|
|
| awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v tbl="$_tbl" -v chain="$_chain" '
|
|
/^[0-9]+/ && $2 + 0 > 0 {
|
|
ln = $1; pkts = $2; bytes = $3; target = $4
|
|
printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=iptables tbl=%s chain=%s line=%s pkts=%s bytes=%s target=%s\n",
|
|
ts, role, host, ts, tbl, chain, ln, pkts, bytes, target
|
|
}
|
|
' | head -n 8 >> "$WAN_WATCH_LOG" 2>/dev/null || true
|
|
done
|
|
}
|
|
|
|
wan_deep_qdisc() {
|
|
_parent_ts="$1"
|
|
_role="$2"
|
|
_hostn="$3"
|
|
_dev="$4"
|
|
[ -n "$_dev" ] || return 0
|
|
tc -s qdisc show dev "$_dev" 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v dev="$_dev" '
|
|
/qdisc/ {
|
|
printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=qdisc dev=%s detail=%s\n",
|
|
ts, role, host, ts, dev, $0
|
|
}
|
|
/backlog/ {
|
|
printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=qdisc dev=%s backlog=%s\n",
|
|
ts, role, host, ts, dev, $0
|
|
}
|
|
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
|
|
}
|
|
|
|
wan_deep_ss_summary() {
|
|
_parent_ts="$1"
|
|
_role="$2"
|
|
_hostn="$3"
|
|
ss -s 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" '
|
|
/TCP:|UDP:/ {
|
|
gsub(/^[ \t]+/, "")
|
|
printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=ss %s\n", ts, role, host, ts, $0
|
|
}
|
|
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
|
|
}
|
|
|
|
# Full deep capture for one anomaly event.
|
|
wan_deep_diag_run() {
|
|
_parent_ts="$1"
|
|
_role="$2"
|
|
_hostn="$3"
|
|
_triggers="$4"
|
|
_wan_target="$5"
|
|
_fe_target="$6"
|
|
_gw_target="$7"
|
|
_route_wan_dev="$8"
|
|
_route_fe_dev="$9"
|
|
|
|
wan_deep_emit alert "$_parent_ts" "$_role" "$_hostn" \
|
|
"parent_ts=$_parent_ts triggers=\"$_triggers\""
|
|
|
|
wan_deep_host_health "$_parent_ts" "$_role" "$_hostn"
|
|
wan_deep_burst_ping "$_parent_ts" "$_role" "$_hostn" "wan" "$_wan_target"
|
|
wan_deep_burst_ping "$_parent_ts" "$_role" "$_hostn" "fe" "$_fe_target"
|
|
if [ -n "$_gw_target" ] && [ "$_gw_target" != "none" ]; then
|
|
wan_deep_burst_ping "$_parent_ts" "$_role" "$_hostn" "gw" "$_gw_target"
|
|
fi
|
|
|
|
wan_deep_route "$_parent_ts" "$_role" "$_hostn" "wan" "$_wan_target"
|
|
wan_deep_route "$_parent_ts" "$_role" "$_hostn" "fe" "$_fe_target"
|
|
wan_deep_neigh "$_parent_ts" "$_role" "$_hostn" "$_fe_target"
|
|
wan_deep_top_procs "$_parent_ts" "$_role" "$_hostn" 5
|
|
wan_deep_softnet "$_parent_ts" "$_role" "$_hostn"
|
|
wan_deep_ss_summary "$_parent_ts" "$_role" "$_hostn"
|
|
|
|
wan_deep_iface_stats "$_parent_ts" "$_role" "$_hostn" "$_route_wan_dev"
|
|
wan_deep_iface_stats "$_parent_ts" "$_role" "$_hostn" "$_route_fe_dev"
|
|
wan_deep_all_ifaces_nonzero "$_parent_ts" "$_role" "$_hostn"
|
|
|
|
if [ "$_role" = "router" ]; then
|
|
wan_deep_iptables_hot "$_parent_ts" "$_role" "$_hostn"
|
|
wan_deep_qdisc "$_parent_ts" "$_role" "$_hostn" "$_route_wan_dev"
|
|
fi
|
|
|
|
if [ "$_role" = "pi" ]; then
|
|
wan_deep_qdisc "$_parent_ts" "$_role" "$_hostn" "$_route_fe_dev"
|
|
fi
|
|
}
|
|
|
|
# Loaded marker for wan-snapshot.sh
|
|
WAN_DEEP_DIAG_LOADED=1 |