ac/android_cast
1
0
Fork 0
mirror of git://f0xx.org/android_cast synced 2026-07-29 04:18:09 +03:00
Code Issues Packages Projects Releases Wiki Activity

bumping

Browse Source 
Net topology: sirene (P2P), spider (MCU), spider net (SFU), pautina (net, mesh), hydra (hybrid), code names
This commit is contained in:
Anton Afanasyeu
2026-06-12 22:13:28 +02:00
parent ad35a60e6c
commit 9acbdbc433
4 changed files with 564 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
examples/network/README.md
View File

@@ -2,6 +2,52 @@
Scripts for **router**, **FE**, and **external vantage** (e.g. Raspberry Pi) — not BE app hosts. Scripts for **router**, **FE**, and **external vantage** (e.g. Raspberry Pi) — not BE app hosts.
## Auto watchdog (v2.1)
When a cron snapshot exceeds thresholds, `wan-snapshot.sh` appends **alert** + **deep** lines to the same log (`WAN_WATCH_LOG`).
| `kind=` | Meaning |
|---------|---------|
| `snapshot` | Normal 15-min ping row |
| `alert` | Threshold breached; lists `triggers=` |
| `deep` | One parseable fact (`section=host`, `burst`, `iface`, `iptables`, …) |
**Deploy both files:**
```bash
sudo cp examples/network/wan-snapshot.sh examples/network/wan-deep-diag.sh /usr/local/bin/
sudo chmod +x /usr/local/bin/wan-snapshot.sh /usr/local/bin/wan-deep-diag.sh
```
**Env** (`/etc/conf.d/wan-snapshot`):
```sh
WAN_WATCH_LOG=/var/log/wan-watch.log
WAN_SNAPSHOT_LOG=/var/log/wan-watch.log
WAN_SNAPSHOT_WATCH=1
```
**Triggers (defaults):** `*_avg_ms >= 50`, `*_loss >= 1%`, `mdev >= 80`, `conntrack >= 80%`, `neigh_fe` not REACHABLE/DELAY, plus role hints `pi_fe_path` / `router_wan_path`.
**Cooldown:** one deep capture per 14 min (`WAN_WATCH_COOLDOWN_S=840`) to avoid cron pile-up.
**Review after a bad spell:**
```bash
/usr/local/bin/wan-snapshot.sh --alerts 80
/usr/local/bin/wan-snapshot.sh --correlate /var/log/wan-watch.log …
grep 'kind=deep.*section=burst' /var/log/wan-watch.log | tail
grep 'kind=deep.*section=iptables' /var/log/wan-watch.log | tail
```
Example deep lines:
```text
kind=alert ts=2026-06-12T21:45:01+03:00 role=router host=monstro parent_ts=... triggers="wan_avg=535.365,router_wan_path"
kind=deep ts=2026-06-12T21:45:01+03:00 role=router host=monstro parent_ts=... section=burst label=wan target=1.1.1.1 loss=0% avg_ms=520 max_ms=790 mdev_ms=110
kind=deep ts=2026-06-12T21:45:01+03:00 role=router host=monstro parent_ts=... section=iptables tbl=filter chain=FORWARD line=3 pkts=188000 target=ACCEPT
```
## `wan-snapshot.sh` (v2) ## `wan-snapshot.sh` (v2)
Periodic ping comparison plus lightweight host/path context. Periodic ping comparison plus lightweight host/path context.

287
examples/network/wan-deep-diag.sh Normal file
View File

@@ -0,0 +1,287 @@
# wan-deep-diag.sh — parseable deep diagnostics (sourced by wan-snapshot.sh).
# Not meant to be executed directly.
#
# Emits kind=deep lines to WAN_WATCH_LOG (key=value, one fact per line).
wan_deep_load_1m() {
awk '{print $1}' /proc/loadavg 2>/dev/null || echo "n/a"
}
wan_deep_cpu_busy_pct() {
awk '/^cpu / {
idle = $5 + $6; total = 0
for (i = 2; i <= NF; i++) total += $i
if (total > 0) printf "%.0f", 100 * (total - idle) / total
}' /proc/stat 2>/dev/null || echo "n/a"
}
wan_deep_mem_avail_kb() {
awk '/^MemAvailable:/ { print $2 }' /proc/meminfo 2>/dev/null || echo "n/a"
}
wan_deep_conntrack_fields() {
_cur="$(cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null)" || _cur=""
_max="$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null)" || _max=""
if [ -n "$_cur" ] && [ -n "$_max" ] && [ "$_max" -gt 0 ] 2>/dev/null; then
_pct=$((_cur * 100 / _max))
printf 'conntrack_cur=%s conntrack_max=%s conntrack_pct=%s' "$_cur" "$_max" "$_pct"
else
printf 'conntrack_cur=n/a conntrack_max=n/a conntrack_pct=n/a'
fi
}
wan_deep_emit() {
# wan_deep_emit KIND TS ROLE HOST [key=value ...]
_kind="$1"
_ts="$2"
_role="$3"
_host="$4"
shift 4
_line="kind=$_kind ts=$_ts role=$_role host=$_host $*"
_dir="$(dirname "$WAN_WATCH_LOG")"
[ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true
printf '%s\n' "$_line" >> "$WAN_WATCH_LOG"
}
# Burst ping (fast) -> one kind=deep line.
wan_deep_burst_ping() {
_parent_ts="$1"
_role="$2"
_hostn="$3"
_label="$4"
_target="$5"
_count="${6:-5}"
_deadline="${7:-6}"
_out="$(ping -c "$_count" -w "$_deadline" "$_target" 2>/dev/null)" || _out=""
if [ -z "$_out" ]; then
wan_deep_emit deep "$_parent_ts" "$_role" "$_hostn" \
"parent_ts=$_parent_ts section=burst label=$_label target=$_target loss=-1 avg_ms=-1"
return 0
fi
_loss="$(printf '%s\n' "$_out" | awk '/packet loss/ {
for (i = 1; i <= NF; i++) if ($i ~ /%$/) { gsub(/%/, "", $i); print $i; exit }
}')"
_rtt="$(printf '%s\n' "$_out" | sed -n 's/.*= \([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\) ms.*/\1 \2 \3 \4/p' | head -1)"
_min="-1" _avg="-1" _max="-1" _mdev="-1"
if [ -n "$_rtt" ]; then
_min="${_rtt%% *}"
_rest="${_rtt#* }"
_avg="${_rest%% *}"
_rest="${_rest#* }"
_max="${_rest%% *}"
_mdev="${_rest#* }"
fi
[ -n "$_loss" ] || _loss="-1"
wan_deep_emit deep "$_parent_ts" "$_role" "$_hostn" \
"parent_ts=$_parent_ts section=burst label=$_label target=$_target loss=${_loss}% min_ms=$_min avg_ms=$_avg max_ms=$_max mdev_ms=$_mdev"
}
wan_deep_host_health() {
_parent_ts="$1"
_role="$2"
_hostn="$3"
_ct="$(wan_deep_conntrack_fields)"
wan_deep_emit deep "$_parent_ts" "$_role" "$_hostn" \
"parent_ts=$_parent_ts section=host load1=$(wan_deep_load_1m) cpu_busy=$(wan_deep_cpu_busy_pct)% mem_avail_kb=$(wan_deep_mem_avail_kb) $_ct"
}
wan_deep_top_procs() {
_parent_ts="$1"
_role="$2"
_hostn="$3"
_n="${4:-5}"
ps -eo pcpu=,pmem=,comm= --sort=-pcpu 2>/dev/null | head -n "$_n" | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v n="$_n" '
NF >= 3 {
rank = NR
printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=proc rank=%d cpu=%s mem=%s comm=%s\n",
ts, role, host, ts, rank, $1, $2, $3
}
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
}
wan_deep_iface_stats() {
_parent_ts="$1"
_role="$2"
_hostn="$3"
_dev="$4"
[ -n "$_dev" ] || return 0
ip -s link show "$_dev" 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v dev="$_dev" '
/^[0-9]+:/ { rx_b = tx_b = rx_p = tx_p = rx_drop = tx_drop = rx_err = tx_err = 0 }
/RX: bytes/ { rx_b = $3; rx_p = $2 }
/RX:.*packets/ { getline; rx_err = $3; rx_drop = $4 }
/TX: bytes/ { tx_b = $3; tx_p = $2 }
/TX:.*packets/ { getline; tx_err = $3; tx_drop = $4 }
END {
printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=iface dev=%s rx_bytes=%s tx_bytes=%s rx_drop=%s tx_drop=%s rx_err=%s tx_err=%s\n",
ts, role, host, ts, dev, rx_b, tx_b, rx_drop, tx_drop, rx_err, tx_err
}
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
}
wan_deep_all_ifaces_nonzero() {
_parent_ts="$1"
_role="$2"
_hostn="$3"
ip -s link 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" '
/^[0-9]+:/ {
if (iface != "" && iface !~ /^lo$/ && (rx_drop + tx_drop + rx_err + tx_err) > 0)
printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=iface dev=%s rx_drop=%s tx_drop=%s rx_err=%s tx_err=%s\n",
ts, role, host, ts, iface, rx_drop, tx_drop, rx_err, tx_err
iface = $2; gsub(":", "", iface)
rx_drop = tx_drop = rx_err = tx_err = 0
}
/RX:/ { getline; rx_err = $3; rx_drop = $4 }
/TX:/ { getline; tx_err = $3; tx_drop = $4 }
END {
if (iface != "" && iface !~ /^lo$/ && (rx_drop + tx_drop + rx_err + tx_err) > 0)
printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=iface dev=%s rx_drop=%s tx_drop=%s rx_err=%s tx_err=%s\n",
ts, role, host, ts, iface, rx_drop, tx_drop, rx_err, tx_err
}
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
}
wan_deep_softnet() {
_parent_ts="$1"
_role="$2"
_hostn="$3"
[ -r /proc/net/softnet_stat ] || return 0
awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" '
$2 + 0 > 0 {
printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=softnet cpu=%d dropped=%s processed=%s\n",
ts, role, host, ts, NR - 1, $2, $1
}
' /proc/net/softnet_stat >> "$WAN_WATCH_LOG" 2>/dev/null || true
}
wan_deep_neigh() {
_parent_ts="$1"
_role="$2"
_hostn="$3"
_ip="$4"
[ -n "$_ip" ] || return 0
ip neigh show "$_ip" 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v ip="$_ip" '
NF >= 5 {
printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=neigh ip=%s lladdr=%s dev=%s state=%s\n",
ts, role, host, ts, ip, $5, $3, $6
}
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
}
wan_deep_route() {
_parent_ts="$1"
_role="$2"
_hostn="$3"
_label="$4"
_target="$5"
[ -n "$_target" ] || return 0
ip route get "$_target" 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v label="$_label" -v target="$_target" '
{
dev = via = ""; src = ""
for (i = 1; i <= NF; i++) {
if ($i == "dev") dev = $(i + 1)
if ($i == "via") via = $(i + 1)
if ($i == "src") src = $(i + 1)
}
printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=route label=%s target=%s dev=%s via=%s src=%s\n",
ts, role, host, ts, label, target, dev, via, src
}
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
}
wan_deep_iptables_hot() {
_parent_ts="$1"
_role="$2"
_hostn="$3"
command -v iptables >/dev/null 2>&1 || return 0
for _spec in "filter:FORWARD" "nat:PREROUTING" "nat:POSTROUTING"; do
_tbl="${_spec%%:*}"
_chain="${_spec#*:}"
iptables -t "$_tbl" -L "$_chain" -n -v --line-numbers 2>/dev/null \
| awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v tbl="$_tbl" -v chain="$_chain" '
/^[0-9]+/ && $2 + 0 > 0 {
ln = $1; pkts = $2; bytes = $3; target = $4
printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=iptables tbl=%s chain=%s line=%s pkts=%s bytes=%s target=%s\n",
ts, role, host, ts, tbl, chain, ln, pkts, bytes, target
}
' | head -n 8 >> "$WAN_WATCH_LOG" 2>/dev/null || true
done
}
wan_deep_qdisc() {
_parent_ts="$1"
_role="$2"
_hostn="$3"
_dev="$4"
[ -n "$_dev" ] || return 0
tc -s qdisc show dev "$_dev" 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" -v dev="$_dev" '
/qdisc/ {
printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=qdisc dev=%s detail=%s\n",
ts, role, host, ts, dev, $0
}
/backlog/ {
printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=qdisc dev=%s backlog=%s\n",
ts, role, host, ts, dev, $0
}
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
}
wan_deep_ss_summary() {
_parent_ts="$1"
_role="$2"
_hostn="$3"
ss -s 2>/dev/null | awk -v ts="$_parent_ts" -v role="$_role" -v host="$_hostn" '
/TCP:|UDP:/ {
gsub(/^[ \t]+/, "")
printf "kind=deep ts=%s role=%s host=%s parent_ts=%s section=ss %s\n", ts, role, host, ts, $0
}
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
}
# Full deep capture for one anomaly event.
wan_deep_diag_run() {
_parent_ts="$1"
_role="$2"
_hostn="$3"
_triggers="$4"
_wan_target="$5"
_fe_target="$6"
_gw_target="$7"
_route_wan_dev="$8"
_route_fe_dev="$9"
wan_deep_emit alert "$_parent_ts" "$_role" "$_hostn" \
"parent_ts=$_parent_ts triggers=\"$_triggers\""
wan_deep_host_health "$_parent_ts" "$_role" "$_hostn"
wan_deep_burst_ping "$_parent_ts" "$_role" "$_hostn" "wan" "$_wan_target"
wan_deep_burst_ping "$_parent_ts" "$_role" "$_hostn" "fe" "$_fe_target"
if [ -n "$_gw_target" ] && [ "$_gw_target" != "none" ]; then
wan_deep_burst_ping "$_parent_ts" "$_role" "$_hostn" "gw" "$_gw_target"
fi
wan_deep_route "$_parent_ts" "$_role" "$_hostn" "wan" "$_wan_target"
wan_deep_route "$_parent_ts" "$_role" "$_hostn" "fe" "$_fe_target"
wan_deep_neigh "$_parent_ts" "$_role" "$_hostn" "$_fe_target"
wan_deep_top_procs "$_parent_ts" "$_role" "$_hostn" 5
wan_deep_softnet "$_parent_ts" "$_role" "$_hostn"
wan_deep_ss_summary "$_parent_ts" "$_role" "$_hostn"
wan_deep_iface_stats "$_parent_ts" "$_role" "$_hostn" "$_route_wan_dev"
wan_deep_iface_stats "$_parent_ts" "$_role" "$_hostn" "$_route_fe_dev"
wan_deep_all_ifaces_nonzero "$_parent_ts" "$_role" "$_hostn"
if [ "$_role" = "router" ]; then
wan_deep_iptables_hot "$_parent_ts" "$_role" "$_hostn"
wan_deep_qdisc "$_parent_ts" "$_role" "$_hostn" "$_route_wan_dev"
fi
if [ "$_role" = "pi" ]; then
wan_deep_qdisc "$_parent_ts" "$_role" "$_hostn" "$_route_fe_dev"
fi
}
# Loaded marker for wan-snapshot.sh
WAN_DEEP_DIAG_LOADED=1

37
examples/network/wan-snapshot.env.example
View File

@@ -1,27 +1,36 @@
# Copy to /etc/conf.d/wan-snapshot on each host (optional). # Copy to /etc/conf.d/wan-snapshot on each host (optional).
# Source from cron: . /etc/conf.d/wan-snapshot 2>/dev/null; /usr/local/bin/wan-snapshot.sh # Cron: . /etc/conf.d/wan-snapshot 2>/dev/null; /usr/local/bin/wan-snapshot.sh >>/var/log/wan-quality.cron.log 2>&1
#
# Deploy BOTH scripts:
# cp wan-snapshot.sh wan-deep-diag.sh /usr/local/bin/
# --- Raspberry Pi (external / home vantage) --- # Unified log (snapshot + auto alert/deep on anomaly):
WAN_WATCH_LOG=/var/log/wan-watch.log
# Legacy alias — if same as WAN_WATCH_LOG, only one file is written:
WAN_SNAPSHOT_LOG=/var/log/wan-watch.log
WAN_SNAPSHOT_WATCH=1
WAN_WATCH_COOLDOWN_S=840
# --- Raspberry Pi ---
# WAN_SNAPSHOT_ROLE=pi # WAN_SNAPSHOT_ROLE=pi
# WAN_SNAPSHOT_LOG=/var/log/wan-quality.log
# ROUTER_PING_TARGET=10.7.6.228 # ROUTER_PING_TARGET=10.7.6.228
# FE_PING_TARGET=10.7.0.10 # FE_PING_TARGET=10.7.0.10
# BE_PING_TARGET=10.7.16.128 # BE_PING_TARGET=10.7.16.128
# --- Debian edge (monstro) ---
# WAN_SNAPSHOT_ROLE=router
# FE_PING_TARGET=10.7.0.10
# BE_PING_TARGET=10.7.16.128
# --- Gentoo FE --- # --- Gentoo FE ---
# WAN_SNAPSHOT_ROLE=fe # WAN_SNAPSHOT_ROLE=fe
# WAN_SNAPSHOT_LOG=/var/log/wan-quality.log
# FE_PING_TARGET=10.7.0.10 # FE_PING_TARGET=10.7.0.10
# BE_PING_TARGET=10.7.16.128 # BE_PING_TARGET=10.7.16.128
# ROUTER_PING_TARGET=10.7.6.228 # ROUTER_PING_TARGET=10.7.6.228
# --- Debian edge router (monstro) --- # Thresholds (ms / %)
# WAN_SNAPSHOT_ROLE=router # WAN_ALERT_MS=50
# WAN_SNAPSHOT_LOG=/var/log/wan-quality.log # FE_ALERT_MS=50
# FE_PING_TARGET=10.7.0.10 # WAN_MDEV_ALERT_MS=80
# BE_PING_TARGET=10.7.16.128 # WAN_CONNTRACK_ALERT_PCT=80
# Shared
# WAN_SNAPSHOT_EXTRAS=1
# PING_COUNT=10
# PING_DEADLINE_S=15

234
examples/network/wan-snapshot.sh
View File

@@ -15,24 +15,40 @@
# */15 * * * * root /usr/local/bin/wan-snapshot.sh >>/var/log/wan-quality.cron.log 2>&1 # */15 * * * * root /usr/local/bin/wan-snapshot.sh >>/var/log/wan-quality.cron.log 2>&1
# #
# Env overrides: # Env overrides:
# WAN_SNAPSHOT_LOG=/var/log/wan-quality.log # WAN_WATCH_LOG=/var/log/wan-watch.log (unified: snapshot + alert + deep)
# WAN_SNAPSHOT_ROLE=auto|router|fe|pi (default auto from hostname) # WAN_SNAPSHOT_LOG=/var/log/wan-quality.log (legacy snapshot copy if different)
# WAN_SNAPSHOT_EXTRAS=1 (0 = ping only, compact v1-style line) # WAN_SNAPSHOT_ROLE=auto|router|fe|pi
# WAN_SNAPSHOT_EXTRAS=1
# WAN_SNAPSHOT_WATCH=1 (0 = disable auto deep-diag on anomaly)
# WAN_SNAPSHOT_LOCK=/var/run/wan-snapshot.lock # WAN_SNAPSHOT_LOCK=/var/run/wan-snapshot.lock
# WAN_PING_TARGET=1.1.1.1 # WAN_WATCH_COOLDOWN_S=840 (min seconds between deep captures)
# FE_PING_TARGET=10.7.0.10 # WAN_ALERT_MS=50 FE_ALERT_MS=50 BE_ALERT_MS=50 GW_ALERT_MS=50
# BE_PING_TARGET=10.7.16.128 (set to "none" to skip) # WAN_LOSS_ALERT_PCT=1 WAN_MDEV_ALERT_MS=80 WAN_CONNTRACK_ALERT_PCT=80
# ROUTER_PING_TARGET=none|10.7.6.228 (optional; useful on Pi) # WAN_PING_TARGET=1.1.1.1 FE_PING_TARGET=10.7.0.10
# GW_PING_TARGET=auto|none|<ip> (default auto = default gateway) # BE_PING_TARGET=10.7.16.128 ROUTER_PING_TARGET=none GW_PING_TARGET=auto
# PING_COUNT=10 # PING_COUNT=10 PING_DEADLINE_S=15
# PING_DEADLINE_S=15 #
# Install deep-diag library next to this script:
# cp wan-snapshot.sh wan-deep-diag.sh /usr/local/bin/
# #
set -eu set -eu
WAN_SNAPSHOT_LOG="${WAN_SNAPSHOT_LOG:-/var/log/wan-quality.log}" WAN_SNAPSHOT_LOG="${WAN_SNAPSHOT_LOG:-/var/log/wan-quality.log}"
WAN_WATCH_LOG="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
WAN_SNAPSHOT_ROLE="${WAN_SNAPSHOT_ROLE:-auto}" WAN_SNAPSHOT_ROLE="${WAN_SNAPSHOT_ROLE:-auto}"
WAN_SNAPSHOT_EXTRAS="${WAN_SNAPSHOT_EXTRAS:-1}" WAN_SNAPSHOT_EXTRAS="${WAN_SNAPSHOT_EXTRAS:-1}"
WAN_SNAPSHOT_WATCH="${WAN_SNAPSHOT_WATCH:-1}"
WAN_SNAPSHOT_LOCK="${WAN_SNAPSHOT_LOCK:-/var/run/wan-snapshot.lock}" WAN_SNAPSHOT_LOCK="${WAN_SNAPSHOT_LOCK:-/var/run/wan-snapshot.lock}"
WAN_WATCH_COOLDOWN_S="${WAN_WATCH_COOLDOWN_S:-840}"
WAN_WATCH_COOLDOWN_FILE="${WAN_WATCH_COOLDOWN_FILE:-/var/run/wan-watch-deep.stamp}"
WAN_ALERT_MS="${WAN_ALERT_MS:-50}"
FE_ALERT_MS="${FE_ALERT_MS:-50}"
BE_ALERT_MS="${BE_ALERT_MS:-50}"
GW_ALERT_MS="${GW_ALERT_MS:-50}"
ROUTER_ALERT_MS="${ROUTER_ALERT_MS:-50}"
WAN_LOSS_ALERT_PCT="${WAN_LOSS_ALERT_PCT:-1}"
WAN_MDEV_ALERT_MS="${WAN_MDEV_ALERT_MS:-80}"
WAN_CONNTRACK_ALERT_PCT="${WAN_CONNTRACK_ALERT_PCT:-80}"
WAN_PING_TARGET="${WAN_PING_TARGET:-1.1.1.1}" WAN_PING_TARGET="${WAN_PING_TARGET:-1.1.1.1}"
FE_PING_TARGET="${FE_PING_TARGET:-10.7.0.10}" FE_PING_TARGET="${FE_PING_TARGET:-10.7.0.10}"
BE_PING_TARGET="${BE_PING_TARGET-10.7.16.128}" BE_PING_TARGET="${BE_PING_TARGET-10.7.16.128}"
@@ -41,6 +57,13 @@ GW_PING_TARGET="${GW_PING_TARGET:-auto}"
PING_COUNT="${PING_COUNT:-10}" PING_COUNT="${PING_COUNT:-10}"
PING_DEADLINE_S="${PING_DEADLINE_S:-15}" PING_DEADLINE_S="${PING_DEADLINE_S:-15}"
_WAN_SNAPSHOT_DIR="$(CDPATH= cd -- "$(dirname "$0")" && pwd)"
# shellcheck source=/dev/null
. "$_WAN_SNAPSHOT_DIR/wan-deep-diag.sh" 2>/dev/null \
|| . /usr/local/bin/wan-deep-diag.sh 2>/dev/null \
|| . /usr/local/lib/wan-deep-diag.sh 2>/dev/null \
|| true
usage() { usage() {
sed -n '2,28p' "$0" sed -n '2,28p' "$0"
printf '\nCommands:\n' printf '\nCommands:\n'
@@ -48,6 +71,7 @@ usage() {
printf ' --report summarize today vs yesterday\n' printf ' --report summarize today vs yesterday\n'
printf ' --tail [N] print last N log lines (default 24)\n' printf ' --tail [N] print last N log lines (default 24)\n'
printf ' --correlate FILE… merge logs by 15-min slot (role= or path basename)\n' printf ' --correlate FILE… merge logs by 15-min slot (role= or path basename)\n'
printf ' --alerts [N] recent kind=alert + kind=deep lines (default 40)\n'
printf ' --help this text\n' printf ' --help this text\n'
} }
@@ -182,12 +206,149 @@ iface_drops_compact() {
' | sed 's/ $//' ' | sed 's/ $//'
} }
# True if line is a snapshot row (legacy ts=… or kind=snapshot).
is_snapshot_line() {
case "$1" in
kind=snapshot*) return 0 ;;
ts=*) return 0 ;;
*) return 1 ;;
esac
}
num_ge() {
_a="$1"
_b="$2"
[ "$_a" != "-1" ] && [ -n "$_a" ] && awk -v a="$_a" -v b="$_b" 'BEGIN { exit (a + 0 >= b + 0) ? 0 : 1 }'
}
watch_cooldown_active() {
[ -r "$WAN_WATCH_COOLDOWN_FILE" ] || return 1
_last="$(cat "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null)" || return 1
_now="$(date +%s 2>/dev/null)" || return 1
[ "$((_now - _last))" -lt "$WAN_WATCH_COOLDOWN_S" ] 2>/dev/null
}
watch_cooldown_stamp() {
date +%s > "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null || true
}
# Returns trigger list (comma-separated) or empty.
detect_anomalies() {
_line="$1"
_role="$2"
_triggers=""
for _pfx in wan fe be gw router; do
_avg="$(field_value "$_line" "${_pfx}_avg_ms")"
_loss="$(field_value "$_line" "${_pfx}_loss")"
_mdev="$(field_value "$_line" "${_pfx}_mdev_ms")"
_max="$(field_value "$_line" "${_pfx}_max_ms")"
_thr="$FE_ALERT_MS"
case "$_pfx" in
wan) _thr="$WAN_ALERT_MS" ;;
fe) _thr="$FE_ALERT_MS" ;;
be) _thr="$BE_ALERT_MS" ;;
gw) _thr="$GW_ALERT_MS" ;;
router) _thr="$ROUTER_ALERT_MS" ;;
esac
if num_ge "$_avg" "$_thr"; then
_triggers="${_triggers}${_triggers:+,}${_pfx}_avg=${_avg}"
fi
if num_ge "$_loss" "$WAN_LOSS_ALERT_PCT"; then
_triggers="${_triggers}${_triggers:+,}${_pfx}_loss=${_loss}"
fi
if num_ge "$_mdev" "$WAN_MDEV_ALERT_MS"; then
_triggers="${_triggers}${_triggers:+,}${_pfx}_mdev=${_mdev}"
fi
if num_ge "$_max" "$((_thr * 3))"; then
_triggers="${_triggers}${_triggers:+,}${_pfx}_max=${_max}"
fi
done
_neigh="$(printf '%s\n' "$_line" | sed -n 's/.*neigh_fe=\([^ ]*\).*/\1/p')"
if [ -n "$_neigh" ] && [ "$_neigh" != "REACHABLE" ] && [ "$_neigh" != "DELAY" ]; then
_triggers="${_triggers}${_triggers:+,}neigh_fe=${_neigh}"
fi
_ct_pct="$(printf '%s\n' "$_line" | sed -n 's/.*(\([0-9]*\)%).*/\1/p' | head -1)"
if [ -n "$_ct_pct" ] && num_ge "$_ct_pct" "$WAN_CONNTRACK_ALERT_PCT"; then
_triggers="${_triggers}${_triggers:+,}conntrack_pct=${_ct_pct}"
fi
# Pi: WAN ok but FE slow -> path anomaly (even if FE just above threshold).
if [ "$_role" = "pi" ]; then
_wan_avg="$(field_value "$_line" "wan_avg_ms")"
_fe_avg="$(field_value "$_line" "fe_avg_ms")"
if num_ge "$_fe_avg" "$FE_ALERT_MS" && ! num_ge "$_wan_avg" "$WAN_ALERT_MS"; then
case "$_triggers" in *fe_avg=*) ;; *)
_triggers="${_triggers}${_triggers:+,}pi_fe_path" ;;
esac
fi
fi
# Router: WAN bad, FE ok -> edge WAN leg (matches monstro xenbr0 pattern).
if [ "$_role" = "router" ]; then
_wan_avg="$(field_value "$_line" "wan_avg_ms")"
_fe_avg="$(field_value "$_line" "fe_avg_ms")"
if num_ge "$_wan_avg" "$WAN_ALERT_MS" && ! num_ge "$_fe_avg" "$FE_ALERT_MS"; then
case "$_triggers" in *wan_avg=*) ;; *)
_triggers="${_triggers}${_triggers:+,}router_wan_path" ;;
esac
fi
fi
printf '%s' "$_triggers"
}
maybe_deep_diag() {
_line="$1"
_ts="$2"
_role="$3"
_hostn="$4"
[ "$WAN_SNAPSHOT_WATCH" = "1" ] || return 0
[ "${WAN_DEEP_DIAG_LOADED:-0}" = "1" ] || {
echo "wan-snapshot: watch enabled but wan-deep-diag.sh not loaded (install alongside wan-snapshot.sh)" >&2
return 0
}
_triggers="$(detect_anomalies "$_line" "$_role")"
[ -n "$_triggers" ] || return 0
if watch_cooldown_active; then
wan_deep_emit alert "$_ts" "$_role" "$_hostn" \
"parent_ts=$_ts triggers=\"$_triggers\" action=skipped_cooldown"
return 0
fi
_gw_target="$GW_PING_TARGET"
if [ "$_gw_target" = "auto" ]; then
_gw_target="$(default_gw)"
fi
_wdev="$(route_dev "$WAN_PING_TARGET")"
_fdev="$(route_dev "$FE_PING_TARGET")"
wan_deep_diag_run "$_ts" "$_role" "$_hostn" "$_triggers" \
"$WAN_PING_TARGET" "$FE_PING_TARGET" "$_gw_target" "$_wdev" "$_fdev"
watch_cooldown_stamp
}
append_watch_log() {
_line="$1"
_dir="$(dirname "$WAN_WATCH_LOG")"
[ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true
printf '%s\n' "$_line" >> "$WAN_WATCH_LOG"
}
acquire_lock() { acquire_lock() {
command -v flock >/dev/null 2>&1 || return 0 command -v flock >/dev/null 2>&1 || return 0
_lock="$WAN_SNAPSHOT_LOCK" _lock="$WAN_SNAPSHOT_LOCK"
_dir="$(dirname "$_lock")" _dir="$(dirname "$_lock")"
if ! mkdir -p "$_dir" 2>/dev/null || ! : >"$_lock" 2>/dev/null; then _fallback="${TMPDIR:-/tmp}/wan-snapshot.$(id -u 2>/dev/null || echo 0).lock"
_lock="${TMPDIR:-/tmp}/wan-snapshot.$(id -u 2>/dev/null || echo 0).lock" if [ ! -d "$_dir" ] || [ ! -w "$_dir" ] 2>/dev/null; then
_lock="$_fallback"
elif ! ( : >>"$_lock" ) 2>/dev/null; then
_lock="$_fallback"
fi fi
exec 9>"$_lock" 2>/dev/null || return 0 exec 9>"$_lock" 2>/dev/null || return 0
flock -n 9 || { flock -n 9 || {
@@ -204,7 +365,7 @@ snapshot_once() {
_role="$(detect_role)" _role="$(detect_role)"
_host="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)" _host="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)"
_line="ts=$_ts ver=2 role=$_role host=$_host" _line="kind=snapshot ts=$_ts ver=2 role=$_role host=$_host"
_line="$(append_ping_fields "$_line" "wan" "$WAN_PING_TARGET")" _line="$(append_ping_fields "$_line" "wan" "$WAN_PING_TARGET")"
_line="$(append_ping_fields "$_line" "fe" "$FE_PING_TARGET")" _line="$(append_ping_fields "$_line" "fe" "$FE_PING_TARGET")"
@@ -241,9 +402,15 @@ snapshot_once() {
[ -n "$_drops" ] && _line="$_line drops=\"$_drops\"" [ -n "$_drops" ] && _line="$_line drops=\"$_drops\""
fi fi
_dir="$(dirname "$WAN_SNAPSHOT_LOG")" append_watch_log "$_line"
[ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true if [ "$WAN_SNAPSHOT_LOG" != "$WAN_WATCH_LOG" ]; then
printf '%s\n' "$_line" >> "$WAN_SNAPSHOT_LOG" _dir="$(dirname "$WAN_SNAPSHOT_LOG")"
[ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true
printf '%s\n' "$_line" >> "$WAN_SNAPSHOT_LOG"
fi
maybe_deep_diag "$_line" "$_ts" "$_role" "$_host"
printf '%s\n' "$_line" printf '%s\n' "$_line"
} }
@@ -271,7 +438,7 @@ day_stats() {
_field="$3" _field="$3"
[ -r "$_log" ] || return 0 [ -r "$_log" ] || return 0
awk -v d="$_date" -v f="$_field" ' awk -v d="$_date" -v f="$_field" '
$0 ~ "^ts=" d { ($0 ~ "^kind=snapshot ts=" d || ($0 ~ "^ts=" d && $0 !~ / kind=/)) {
for (i = 1; i <= NF; i++) { for (i = 1; i <= NF; i++) {
if ($i ~ "^" f "=") { if ($i ~ "^" f "=") {
split($i, a, "=") split($i, a, "=")
@@ -302,7 +469,7 @@ day_max_loss() {
_field="$3" _field="$3"
[ -r "$_log" ] || return 0 [ -r "$_log" ] || return 0
awk -v d="$_date" -v f="$_field" ' awk -v d="$_date" -v f="$_field" '
$0 ~ "^ts=" d { ($0 ~ "^kind=snapshot ts=" d || ($0 ~ "^ts=" d && $0 !~ / kind=/)) {
for (i = 1; i <= NF; i++) { for (i = 1; i <= NF; i++) {
if ($i ~ "^" f "=") { if ($i ~ "^" f "=") {
split($i, a, "=") split($i, a, "=")
@@ -323,7 +490,7 @@ report_day_block() {
_log="$1" _log="$1"
_label="$2" _label="$2"
_date="$3" _date="$3"
_samples="$(grep -c "^ts=$_date" "$_log" 2>/dev/null || echo 0)" _samples="$(grep -cE "^kind=snapshot ts=$_date|^ts=$_date" "$_log" 2>/dev/null || echo 0)"
printf '%s=%s samples=%s\n' "$_label" "$_date" "$_samples" printf '%s=%s samples=%s\n' "$_label" "$_date" "$_samples"
for _pfx in wan fe be router gw; do for _pfx in wan fe be router gw; do
_stats="$(day_stats "$_log" "$_date" "${_pfx}_avg_ms")" _stats="$(day_stats "$_log" "$_date" "${_pfx}_avg_ms")"
@@ -344,7 +511,7 @@ report_day_block() {
} }
report() { report() {
_log="$WAN_SNAPSHOT_LOG" _log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
_today="$(date '+%Y-%m-%d')" _today="$(date '+%Y-%m-%d')"
_yesterday="$(date -d 'yesterday' '+%Y-%m-%d' 2>/dev/null || date -v-1d '+%Y-%m-%d' 2>/dev/null || echo '')" _yesterday="$(date -d 'yesterday' '+%Y-%m-%d' 2>/dev/null || date -v-1d '+%Y-%m-%d' 2>/dev/null || echo '')"
@@ -357,7 +524,7 @@ report() {
printf 'log=%s role=%s\n' "$_log" "$_role" printf 'log=%s role=%s\n' "$_log" "$_role"
report_day_block "$_log" "today" "$_today" report_day_block "$_log" "today" "$_today"
if [ -n "$_yesterday" ] && grep -q "^ts=$_yesterday" "$_log" 2>/dev/null; then if [ -n "$_yesterday" ] && grep -qE "^kind=snapshot ts=$_yesterday|^ts=$_yesterday" "$_log" 2>/dev/null; then
report_day_block "$_log" "yesterday" "$_yesterday" report_day_block "$_log" "yesterday" "$_yesterday"
fi fi
@@ -373,11 +540,22 @@ report() {
tail_log() { tail_log() {
_n="${1:-24}" _n="${1:-24}"
[ -r "$WAN_SNAPSHOT_LOG" ] || { _log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
echo "wan-snapshot: no log at $WAN_SNAPSHOT_LOG" >&2 [ -r "$_log" ] || {
echo "wan-snapshot: no log at $_log" >&2
exit 1 exit 1
} }
tail -n "$_n" "$WAN_SNAPSHOT_LOG" grep -E '^kind=snapshot |^ts=' "$_log" 2>/dev/null | tail -n "$_n"
}
tail_alerts() {
_n="${1:-40}"
_log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
[ -r "$_log" ] || {
echo "wan-snapshot: no log at $_log" >&2
exit 1
}
grep -E '^kind=alert |^kind=deep ' "$_log" 2>/dev/null | tail -n "$_n"
} }
# Guess role label for correlate when log line has no role=. # Guess role label for correlate when log line has no role=.
@@ -428,11 +606,12 @@ correlate_logs() {
} }
_label="$(basename "$_path")" _label="$(basename "$_path")"
while IFS= read -r _line || [ -n "$_line" ]; do while IFS= read -r _line || [ -n "$_line" ]; do
is_snapshot_line "$_line" || continue
case "$_line" in case "$_line" in
ts=*) ;; kind=snapshot*) _ts="${_line#kind=snapshot ts=}" ;;
ts=*) _ts="${_line#ts=}" ;;
*) continue ;; *) continue ;;
esac esac
_ts="${_line#ts=}"
_ts="${_ts%% *}" _ts="${_ts%% *}"
_slot="$(slot_key "$_ts")" _slot="$(slot_key "$_ts")"
_role="$(log_role_label "$_path" "$_line")" _role="$(log_role_label "$_path" "$_line")"
@@ -500,6 +679,9 @@ case "${1:-}" in
--tail|tail) --tail|tail)
tail_log "${2:-24}" tail_log "${2:-24}"
;; ;;
--alerts|alerts)
tail_alerts "${2:-40}"
;;
--correlate|correlate) --correlate|correlate)
shift shift
correlate_logs "$@" correlate_logs "$@"