mirror of
git://f0xx.org/android_cast
synced 2026-07-29 05:17:39 +03:00
sync on be, vpn
This commit is contained in:
Anton Afanasyeu
@@ -1,696 +1,4 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Log WAN vs LAN ping quality from any vantage (router, FE, Pi, …).
|
||||
# Use to compare "rocketship" evenings vs slow periods (ISP/upstream vs local).
|
||||
#
|
||||
# Topology (see docs/INFRA.md):
|
||||
# world -> 134.17.26.161 (router 10.7.6.228) -> 10.7.0.10 (FE) -> 10.7.0.0/8 (BE, …)
|
||||
#
|
||||
# Install:
|
||||
# sudo cp wan-snapshot.sh /usr/local/bin/wan-snapshot.sh
|
||||
# sudo chmod +x /usr/local/bin/wan-snapshot.sh
|
||||
# sudo mkdir -p /var/log
|
||||
#
|
||||
# Cron (every 15 minutes — ONE entry only; script uses flock to skip overlaps):
|
||||
# */15 * * * * root /usr/local/bin/wan-snapshot.sh >>/var/log/wan-quality.cron.log 2>&1
|
||||
#
|
||||
# Env overrides:
|
||||
# WAN_WATCH_LOG=/var/log/wan-watch.log (unified: snapshot + alert + deep)
|
||||
# WAN_SNAPSHOT_LOG=/var/log/wan-quality.log (legacy snapshot copy if different)
|
||||
# WAN_SNAPSHOT_ROLE=auto|router|fe|pi
|
||||
# WAN_SNAPSHOT_EXTRAS=1
|
||||
# WAN_SNAPSHOT_WATCH=1 (0 = disable auto deep-diag on anomaly)
|
||||
# WAN_SNAPSHOT_LOCK=/var/run/wan-snapshot.lock
|
||||
# WAN_WATCH_COOLDOWN_S=840 (min seconds between deep captures)
|
||||
# WAN_ALERT_MS=50 FE_ALERT_MS=50 BE_ALERT_MS=50 GW_ALERT_MS=50
|
||||
# WAN_LOSS_ALERT_PCT=1 WAN_MDEV_ALERT_MS=80 WAN_CONNTRACK_ALERT_PCT=80
|
||||
# WAN_PING_TARGET=1.1.1.1 FE_PING_TARGET=10.7.0.10
|
||||
# BE_PING_TARGET=10.7.16.128 ROUTER_PING_TARGET=none GW_PING_TARGET=auto
|
||||
# PING_COUNT=10 PING_DEADLINE_S=15
|
||||
#
|
||||
# Install deep-diag library next to this script:
|
||||
# cp wan-snapshot.sh wan-deep-diag.sh /usr/local/bin/
|
||||
#
|
||||
set -eu
|
||||
|
||||
WAN_SNAPSHOT_LOG="${WAN_SNAPSHOT_LOG:-/var/log/wan-quality.log}"
|
||||
WAN_WATCH_LOG="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
|
||||
WAN_SNAPSHOT_ROLE="${WAN_SNAPSHOT_ROLE:-auto}"
|
||||
WAN_SNAPSHOT_EXTRAS="${WAN_SNAPSHOT_EXTRAS:-1}"
|
||||
WAN_SNAPSHOT_WATCH="${WAN_SNAPSHOT_WATCH:-1}"
|
||||
WAN_SNAPSHOT_LOCK="${WAN_SNAPSHOT_LOCK:-/var/run/wan-snapshot.lock}"
|
||||
WAN_WATCH_COOLDOWN_S="${WAN_WATCH_COOLDOWN_S:-840}"
|
||||
WAN_WATCH_COOLDOWN_FILE="${WAN_WATCH_COOLDOWN_FILE:-/var/run/wan-watch-deep.stamp}"
|
||||
WAN_ALERT_MS="${WAN_ALERT_MS:-50}"
|
||||
FE_ALERT_MS="${FE_ALERT_MS:-50}"
|
||||
BE_ALERT_MS="${BE_ALERT_MS:-50}"
|
||||
GW_ALERT_MS="${GW_ALERT_MS:-50}"
|
||||
ROUTER_ALERT_MS="${ROUTER_ALERT_MS:-50}"
|
||||
WAN_LOSS_ALERT_PCT="${WAN_LOSS_ALERT_PCT:-1}"
|
||||
WAN_MDEV_ALERT_MS="${WAN_MDEV_ALERT_MS:-80}"
|
||||
WAN_CONNTRACK_ALERT_PCT="${WAN_CONNTRACK_ALERT_PCT:-80}"
|
||||
WAN_PING_TARGET="${WAN_PING_TARGET:-1.1.1.1}"
|
||||
FE_PING_TARGET="${FE_PING_TARGET:-10.7.0.10}"
|
||||
BE_PING_TARGET="${BE_PING_TARGET-10.7.16.128}"
|
||||
ROUTER_PING_TARGET="${ROUTER_PING_TARGET:-none}"
|
||||
GW_PING_TARGET="${GW_PING_TARGET:-auto}"
|
||||
PING_COUNT="${PING_COUNT:-10}"
|
||||
PING_DEADLINE_S="${PING_DEADLINE_S:-15}"
|
||||
|
||||
_WAN_SNAPSHOT_DIR="$(CDPATH= cd -- "$(dirname "$0")" && pwd)"
|
||||
# shellcheck source=/dev/null
|
||||
. "$_WAN_SNAPSHOT_DIR/wan-deep-diag.sh" 2>/dev/null \
|
||||
|| . /usr/local/bin/wan-deep-diag.sh 2>/dev/null \
|
||||
|| . /usr/local/lib/wan-deep-diag.sh 2>/dev/null \
|
||||
|| true
|
||||
|
||||
usage() {
|
||||
sed -n '2,28p' "$0"
|
||||
printf '\nCommands:\n'
|
||||
printf ' (default) append one snapshot line to the log\n'
|
||||
printf ' --report summarize today vs yesterday\n'
|
||||
printf ' --tail [N] print last N log lines (default 24)\n'
|
||||
printf ' --correlate FILE… merge logs by 15-min slot (role= or path basename)\n'
|
||||
printf ' --alerts [N] recent kind=alert + kind=deep lines (default 40)\n'
|
||||
printf ' --help this text\n'
|
||||
}
|
||||
|
||||
need_ping() {
|
||||
command -v ping >/dev/null 2>&1 || {
|
||||
echo "wan-snapshot: missing ping" >&2
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
detect_role() {
|
||||
_role="$WAN_SNAPSHOT_ROLE"
|
||||
if [ "$_role" != "auto" ]; then
|
||||
printf '%s' "$_role"
|
||||
return 0
|
||||
fi
|
||||
_hn="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)"
|
||||
case "$_hn" in
|
||||
*raspberry*|*rpi*) printf 'pi' ;;
|
||||
*monstro*|*router*|*gw*) printf 'router' ;;
|
||||
*gentoo*|*wg0*|*fe*) printf 'fe' ;;
|
||||
*) printf 'host' ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ping_summary HOST -> loss_pct min_ms avg_ms max_ms mdev_ms (-1 for missing)
|
||||
ping_summary() {
|
||||
_host="$1"
|
||||
_out=""
|
||||
_out="$(ping -c "$PING_COUNT" -w "$PING_DEADLINE_S" "$_host" 2>/dev/null)" || true
|
||||
if [ -z "$_out" ]; then
|
||||
printf '%s %s %s %s %s' "-1" "-1" "-1" "-1" "-1"
|
||||
return 0
|
||||
fi
|
||||
_loss="$(printf '%s\n' "$_out" | awk '/packet loss/ {
|
||||
for (i = 1; i <= NF; i++) if ($i ~ /%$/) { gsub(/%/, "", $i); print $i; exit }
|
||||
}')"
|
||||
_min="-1"
|
||||
_avg="-1"
|
||||
_max="-1"
|
||||
_mdev="-1"
|
||||
_rtt="$(printf '%s\n' "$_out" | sed -n 's/.*= \([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\) ms.*/\1 \2 \3 \4/p' | head -1)"
|
||||
if [ -n "$_rtt" ]; then
|
||||
_min="${_rtt%% *}"
|
||||
_rest="${_rtt#* }"
|
||||
_avg="${_rest%% *}"
|
||||
_rest="${_rest#* }"
|
||||
_max="${_rest%% *}"
|
||||
_mdev="${_rest#* }"
|
||||
else
|
||||
_avg="$(printf '%s\n' "$_out" | awk -F'[=/ ]' '/min\/avg\/max/ { print $8; exit }')"
|
||||
[ -n "$_avg" ] || _avg="-1"
|
||||
fi
|
||||
[ -n "$_loss" ] || _loss="-1"
|
||||
printf '%s %s %s %s %s' "$_loss" "$_min" "$_avg" "$_max" "$_mdev"
|
||||
}
|
||||
|
||||
append_ping_fields() {
|
||||
_line="$1"
|
||||
_prefix="$2"
|
||||
_host="$3"
|
||||
_sum="$(ping_summary "$_host")"
|
||||
_loss="${_sum%% *}"
|
||||
_rest="${_sum#* }"
|
||||
_min="${_rest%% *}"
|
||||
_rest="${_rest#* }"
|
||||
_avg="${_rest%% *}"
|
||||
_rest="${_rest#* }"
|
||||
_max="${_rest%% *}"
|
||||
_mdev="${_rest#* }"
|
||||
printf '%s %s=%s %s_loss=%s%% %s_min_ms=%s %s_avg_ms=%s %s_max_ms=%s %s_mdev_ms=%s' \
|
||||
"$_line" "$_prefix" "$_host" "$_prefix" "$_loss" "$_prefix" "$_min" \
|
||||
"$_prefix" "$_avg" "$_prefix" "$_max" "$_prefix" "$_mdev"
|
||||
}
|
||||
|
||||
route_dev() {
|
||||
_host="$1"
|
||||
ip route get "$_host" 2>/dev/null | awk '{for (i = 1; i <= NF; i++) if ($i == "dev") { print $(i + 1); exit }}'
|
||||
}
|
||||
|
||||
route_gw() {
|
||||
_host="$1"
|
||||
ip route get "$_host" 2>/dev/null | awk '{for (i = 1; i <= NF; i++) if ($i == "via") { print $(i + 1); exit }}'
|
||||
}
|
||||
|
||||
default_gw() {
|
||||
ip route show default 2>/dev/null | awk '{print $3; exit}'
|
||||
}
|
||||
|
||||
load_1m() {
|
||||
awk '{print $1}' /proc/loadavg 2>/dev/null || echo "n/a"
|
||||
}
|
||||
|
||||
cpu_busy_pct() {
|
||||
awk '/^cpu / {
|
||||
idle = $5 + $6; total = 0
|
||||
for (i = 2; i <= NF; i++) total += $i
|
||||
if (total > 0) printf "%.0f", 100 * (total - idle) / total
|
||||
}' /proc/stat 2>/dev/null || echo "n/a"
|
||||
}
|
||||
|
||||
conntrack_pct() {
|
||||
_cur="$(cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null)" || _cur=""
|
||||
_max="$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null)" || _max=""
|
||||
if [ -n "$_cur" ] && [ -n "$_max" ] && [ "$_max" -gt 0 ] 2>/dev/null; then
|
||||
_pct=$((_cur * 100 / _max))
|
||||
printf 'conntrack=%s/%s(%s%%)' "$_cur" "$_max" "$_pct"
|
||||
fi
|
||||
}
|
||||
|
||||
neigh_state() {
|
||||
_host="$1"
|
||||
ip neigh show "$_host" 2>/dev/null | awk '{print $6; exit}'
|
||||
}
|
||||
|
||||
iface_drops_compact() {
|
||||
ip -s link 2>/dev/null | awk '
|
||||
/^[0-9]+:/ {
|
||||
if (iface != "" && iface !~ /^lo$/) {
|
||||
if (rx_drop + tx_drop + rx_err + tx_err > 0)
|
||||
printf "%s:rd=%d,td=%d,re=%d,te=%d ", iface, rx_drop, tx_drop, rx_err, tx_err
|
||||
}
|
||||
iface = $2; gsub(":", "", iface)
|
||||
rx_drop = tx_drop = rx_err = tx_err = 0
|
||||
}
|
||||
/RX:/ { getline; rx_err = $3; rx_drop = $4 }
|
||||
/TX:/ { getline; tx_err = $3; tx_drop = $4 }
|
||||
END {
|
||||
if (iface != "" && iface !~ /^lo$/ && rx_drop + tx_drop + rx_err + tx_err > 0)
|
||||
printf "%s:rd=%d,td=%d,re=%d,te=%d", iface, rx_drop, tx_drop, rx_err, tx_err
|
||||
}
|
||||
' | sed 's/ $//'
|
||||
}
|
||||
|
||||
# True if line is a snapshot row (legacy ts=… or kind=snapshot).
|
||||
is_snapshot_line() {
|
||||
case "$1" in
|
||||
kind=snapshot*) return 0 ;;
|
||||
ts=*) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
num_ge() {
|
||||
_a="$1"
|
||||
_b="$2"
|
||||
[ "$_a" != "-1" ] && [ -n "$_a" ] && awk -v a="$_a" -v b="$_b" 'BEGIN { exit (a + 0 >= b + 0) ? 0 : 1 }'
|
||||
}
|
||||
|
||||
watch_cooldown_active() {
|
||||
[ -r "$WAN_WATCH_COOLDOWN_FILE" ] || return 1
|
||||
_last="$(cat "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null)" || return 1
|
||||
_now="$(date +%s 2>/dev/null)" || return 1
|
||||
[ "$((_now - _last))" -lt "$WAN_WATCH_COOLDOWN_S" ] 2>/dev/null
|
||||
}
|
||||
|
||||
watch_cooldown_stamp() {
|
||||
date +%s > "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# Returns trigger list (comma-separated) or empty.
|
||||
detect_anomalies() {
|
||||
_line="$1"
|
||||
_role="$2"
|
||||
_triggers=""
|
||||
|
||||
for _pfx in wan fe be gw router; do
|
||||
_avg="$(field_value "$_line" "${_pfx}_avg_ms")"
|
||||
_loss="$(field_value "$_line" "${_pfx}_loss")"
|
||||
_mdev="$(field_value "$_line" "${_pfx}_mdev_ms")"
|
||||
_max="$(field_value "$_line" "${_pfx}_max_ms")"
|
||||
_thr="$FE_ALERT_MS"
|
||||
case "$_pfx" in
|
||||
wan) _thr="$WAN_ALERT_MS" ;;
|
||||
fe) _thr="$FE_ALERT_MS" ;;
|
||||
be) _thr="$BE_ALERT_MS" ;;
|
||||
gw) _thr="$GW_ALERT_MS" ;;
|
||||
router) _thr="$ROUTER_ALERT_MS" ;;
|
||||
esac
|
||||
if num_ge "$_avg" "$_thr"; then
|
||||
_triggers="${_triggers}${_triggers:+,}${_pfx}_avg=${_avg}"
|
||||
fi
|
||||
if num_ge "$_loss" "$WAN_LOSS_ALERT_PCT"; then
|
||||
_triggers="${_triggers}${_triggers:+,}${_pfx}_loss=${_loss}"
|
||||
fi
|
||||
if num_ge "$_mdev" "$WAN_MDEV_ALERT_MS"; then
|
||||
_triggers="${_triggers}${_triggers:+,}${_pfx}_mdev=${_mdev}"
|
||||
fi
|
||||
if num_ge "$_max" "$((_thr * 3))"; then
|
||||
_triggers="${_triggers}${_triggers:+,}${_pfx}_max=${_max}"
|
||||
fi
|
||||
done
|
||||
|
||||
_neigh="$(printf '%s\n' "$_line" | sed -n 's/.*neigh_fe=\([^ ]*\).*/\1/p')"
|
||||
if [ -n "$_neigh" ] && [ "$_neigh" != "REACHABLE" ] && [ "$_neigh" != "DELAY" ]; then
|
||||
_triggers="${_triggers}${_triggers:+,}neigh_fe=${_neigh}"
|
||||
fi
|
||||
|
||||
_ct_pct="$(printf '%s\n' "$_line" | sed -n 's/.*(\([0-9]*\)%).*/\1/p' | head -1)"
|
||||
if [ -n "$_ct_pct" ] && num_ge "$_ct_pct" "$WAN_CONNTRACK_ALERT_PCT"; then
|
||||
_triggers="${_triggers}${_triggers:+,}conntrack_pct=${_ct_pct}"
|
||||
fi
|
||||
|
||||
# Pi: WAN ok but FE slow -> path anomaly (even if FE just above threshold).
|
||||
if [ "$_role" = "pi" ]; then
|
||||
_wan_avg="$(field_value "$_line" "wan_avg_ms")"
|
||||
_fe_avg="$(field_value "$_line" "fe_avg_ms")"
|
||||
if num_ge "$_fe_avg" "$FE_ALERT_MS" && ! num_ge "$_wan_avg" "$WAN_ALERT_MS"; then
|
||||
case "$_triggers" in *fe_avg=*) ;; *)
|
||||
_triggers="${_triggers}${_triggers:+,}pi_fe_path" ;;
|
||||
esac
|
||||
fi
|
||||
fi
|
||||
|
||||
# Router: WAN bad, FE ok -> edge WAN leg (matches monstro xenbr0 pattern).
|
||||
if [ "$_role" = "router" ]; then
|
||||
_wan_avg="$(field_value "$_line" "wan_avg_ms")"
|
||||
_fe_avg="$(field_value "$_line" "fe_avg_ms")"
|
||||
if num_ge "$_wan_avg" "$WAN_ALERT_MS" && ! num_ge "$_fe_avg" "$FE_ALERT_MS"; then
|
||||
case "$_triggers" in *wan_avg=*) ;; *)
|
||||
_triggers="${_triggers}${_triggers:+,}router_wan_path" ;;
|
||||
esac
|
||||
fi
|
||||
fi
|
||||
|
||||
printf '%s' "$_triggers"
|
||||
}
|
||||
|
||||
maybe_deep_diag() {
|
||||
_line="$1"
|
||||
_ts="$2"
|
||||
_role="$3"
|
||||
_hostn="$4"
|
||||
|
||||
[ "$WAN_SNAPSHOT_WATCH" = "1" ] || return 0
|
||||
[ "${WAN_DEEP_DIAG_LOADED:-0}" = "1" ] || {
|
||||
echo "wan-snapshot: watch enabled but wan-deep-diag.sh not loaded (install alongside wan-snapshot.sh)" >&2
|
||||
return 0
|
||||
}
|
||||
|
||||
_triggers="$(detect_anomalies "$_line" "$_role")"
|
||||
[ -n "$_triggers" ] || return 0
|
||||
|
||||
if watch_cooldown_active; then
|
||||
wan_deep_emit alert "$_ts" "$_role" "$_hostn" \
|
||||
"parent_ts=$_ts triggers=\"$_triggers\" action=skipped_cooldown"
|
||||
return 0
|
||||
fi
|
||||
|
||||
_gw_target="$GW_PING_TARGET"
|
||||
if [ "$_gw_target" = "auto" ]; then
|
||||
_gw_target="$(default_gw)"
|
||||
fi
|
||||
_wdev="$(route_dev "$WAN_PING_TARGET")"
|
||||
_fdev="$(route_dev "$FE_PING_TARGET")"
|
||||
|
||||
wan_deep_diag_run "$_ts" "$_role" "$_hostn" "$_triggers" \
|
||||
"$WAN_PING_TARGET" "$FE_PING_TARGET" "$_gw_target" "$_wdev" "$_fdev"
|
||||
watch_cooldown_stamp
|
||||
}
|
||||
|
||||
append_watch_log() {
|
||||
_line="$1"
|
||||
_dir="$(dirname "$WAN_WATCH_LOG")"
|
||||
[ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true
|
||||
printf '%s\n' "$_line" >> "$WAN_WATCH_LOG"
|
||||
}
|
||||
|
||||
acquire_lock() {
|
||||
command -v flock >/dev/null 2>&1 || return 0
|
||||
_lock="$WAN_SNAPSHOT_LOCK"
|
||||
_dir="$(dirname "$_lock")"
|
||||
_fallback="${TMPDIR:-/tmp}/wan-snapshot.$(id -u 2>/dev/null || echo 0).lock"
|
||||
if [ ! -d "$_dir" ] || [ ! -w "$_dir" ] 2>/dev/null; then
|
||||
_lock="$_fallback"
|
||||
elif ! ( : >>"$_lock" ) 2>/dev/null; then
|
||||
_lock="$_fallback"
|
||||
fi
|
||||
exec 9>"$_lock" 2>/dev/null || return 0
|
||||
flock -n 9 || {
|
||||
echo "wan-snapshot: skip (already running, lock=$_lock)" >&2
|
||||
exit 0
|
||||
}
|
||||
}
|
||||
|
||||
snapshot_once() {
|
||||
need_ping
|
||||
acquire_lock
|
||||
|
||||
_ts="$(date -Iseconds 2>/dev/null || date '+%Y-%m-%dT%H:%M:%S%z')"
|
||||
_role="$(detect_role)"
|
||||
_host="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)"
|
||||
|
||||
_line="kind=snapshot ts=$_ts ver=2 role=$_role host=$_host"
|
||||
_line="$(append_ping_fields "$_line" "wan" "$WAN_PING_TARGET")"
|
||||
_line="$(append_ping_fields "$_line" "fe" "$FE_PING_TARGET")"
|
||||
|
||||
if [ -n "$BE_PING_TARGET" ] && [ "$BE_PING_TARGET" != "none" ]; then
|
||||
_line="$(append_ping_fields "$_line" "be" "$BE_PING_TARGET")"
|
||||
fi
|
||||
|
||||
if [ -n "$ROUTER_PING_TARGET" ] && [ "$ROUTER_PING_TARGET" != "none" ]; then
|
||||
_line="$(append_ping_fields "$_line" "router" "$ROUTER_PING_TARGET")"
|
||||
fi
|
||||
|
||||
_gw_target="$GW_PING_TARGET"
|
||||
if [ "$_gw_target" = "auto" ]; then
|
||||
_gw_target="$(default_gw)"
|
||||
[ -n "$_gw_target" ] || _gw_target="none"
|
||||
fi
|
||||
if [ -n "$_gw_target" ] && [ "$_gw_target" != "none" ]; then
|
||||
_line="$(append_ping_fields "$_line" "gw" "$_gw_target")"
|
||||
fi
|
||||
|
||||
if [ "$WAN_SNAPSHOT_EXTRAS" = "1" ]; then
|
||||
_line="$_line load1=$(load_1m) cpu_busy=$(cpu_busy_pct)%"
|
||||
_ct="$(conntrack_pct)"
|
||||
[ -n "$_ct" ] && _line="$_line $_ct"
|
||||
_wdev="$(route_dev "$WAN_PING_TARGET")"
|
||||
_fdev="$(route_dev "$FE_PING_TARGET")"
|
||||
[ -n "$_wdev" ] && _line="$_line route_wan=$_wdev"
|
||||
[ -n "$_fdev" ] && _line="$_line route_fe=$_fdev"
|
||||
_wgw="$(route_gw "$WAN_PING_TARGET")"
|
||||
[ -n "$_wgw" ] && _line="$_line via_wan=$_wgw"
|
||||
_nfe="$(neigh_state "$FE_PING_TARGET")"
|
||||
[ -n "$_nfe" ] && _line="$_line neigh_fe=$_nfe"
|
||||
_drops="$(iface_drops_compact)"
|
||||
[ -n "$_drops" ] && _line="$_line drops=\"$_drops\""
|
||||
fi
|
||||
|
||||
append_watch_log "$_line"
|
||||
if [ "$WAN_SNAPSHOT_LOG" != "$WAN_WATCH_LOG" ]; then
|
||||
_dir="$(dirname "$WAN_SNAPSHOT_LOG")"
|
||||
[ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true
|
||||
printf '%s\n' "$_line" >> "$WAN_SNAPSHOT_LOG"
|
||||
fi
|
||||
|
||||
maybe_deep_diag "$_line" "$_ts" "$_role" "$_host"
|
||||
|
||||
printf '%s\n' "$_line"
|
||||
}
|
||||
|
||||
# Extract numeric field from log line (supports wan_avg_ms and legacy layouts).
|
||||
field_value() {
|
||||
_line="$1"
|
||||
_field="$2"
|
||||
printf '%s\n' "$_line" | awk -v f="$_field" '
|
||||
{
|
||||
for (i = 1; i <= NF; i++) {
|
||||
if ($i ~ "^" f "=") {
|
||||
split($i, a, "=")
|
||||
v = a[2]
|
||||
gsub(/%/, "", v)
|
||||
if (v + 0 == v) { print v; exit }
|
||||
}
|
||||
}
|
||||
}
|
||||
'
|
||||
}
|
||||
|
||||
day_stats() {
|
||||
_log="$1"
|
||||
_date="$2"
|
||||
_field="$3"
|
||||
[ -r "$_log" ] || return 0
|
||||
awk -v d="$_date" -v f="$_field" '
|
||||
($0 ~ "^kind=snapshot ts=" d || ($0 ~ "^ts=" d && $0 !~ / kind=/)) {
|
||||
for (i = 1; i <= NF; i++) {
|
||||
if ($i ~ "^" f "=") {
|
||||
split($i, a, "=")
|
||||
v = a[2]
|
||||
gsub(/%/, "", v)
|
||||
if (v + 0 == v && v >= 0) {
|
||||
n++; s += v; if (v > max) max = v
|
||||
if (v >= 50) bad++
|
||||
vals[n] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
END {
|
||||
if (n == 0) { print "n/a n/a n/a n/a"; exit }
|
||||
asort(vals)
|
||||
p50 = vals[int((n + 1) * 0.5)]
|
||||
p90 = vals[int(n * 0.9)]
|
||||
if (p90 == "") p90 = vals[n]
|
||||
printf "%.2f %.2f %.2f %.0f %d/%d", s / n, p50, p90, max, bad + 0, n
|
||||
}
|
||||
' "$_log"
|
||||
}
|
||||
|
||||
day_max_loss() {
|
||||
_log="$1"
|
||||
_date="$2"
|
||||
_field="$3"
|
||||
[ -r "$_log" ] || return 0
|
||||
awk -v d="$_date" -v f="$_field" '
|
||||
($0 ~ "^kind=snapshot ts=" d || ($0 ~ "^ts=" d && $0 !~ / kind=/)) {
|
||||
for (i = 1; i <= NF; i++) {
|
||||
if ($i ~ "^" f "=") {
|
||||
split($i, a, "=")
|
||||
v = a[2]
|
||||
gsub(/%/, "", v)
|
||||
if (v + 0 == v && v >= 0 && v > max) max = v
|
||||
}
|
||||
}
|
||||
}
|
||||
END {
|
||||
if (max == "") print "n/a"
|
||||
else printf "%.0f", max
|
||||
}
|
||||
' "$_log"
|
||||
}
|
||||
|
||||
report_day_block() {
|
||||
_log="$1"
|
||||
_label="$2"
|
||||
_date="$3"
|
||||
_samples="$(grep -cE "^kind=snapshot ts=$_date|^ts=$_date" "$_log" 2>/dev/null || echo 0)"
|
||||
printf '%s=%s samples=%s\n' "$_label" "$_date" "$_samples"
|
||||
for _pfx in wan fe be router gw; do
|
||||
_stats="$(day_stats "$_log" "$_date" "${_pfx}_avg_ms")"
|
||||
_mean="${_stats%% *}"
|
||||
_rest="${_stats#* }"
|
||||
_p50="${_rest%% *}"
|
||||
_rest="${_rest#* }"
|
||||
_p90="${_rest%% *}"
|
||||
_rest="${_rest#* }"
|
||||
_max="${_rest%% *}"
|
||||
_bad="${_rest#* }"
|
||||
_loss="$(day_max_loss "$_log" "$_date" "${_pfx}_loss")"
|
||||
if [ "$_mean" != "n/a" ]; then
|
||||
printf ' %s: mean=%s p50=%s p90=%s max=%s bad(>=50ms)=%s loss_max=%s%%\n' \
|
||||
"$_pfx" "$_mean" "$_p50" "$_p90" "$_max" "$_bad" "$_loss"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
report() {
|
||||
_log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
|
||||
_today="$(date '+%Y-%m-%d')"
|
||||
_yesterday="$(date -d 'yesterday' '+%Y-%m-%d' 2>/dev/null || date -v-1d '+%Y-%m-%d' 2>/dev/null || echo '')"
|
||||
|
||||
if [ ! -r "$_log" ]; then
|
||||
echo "wan-snapshot: no log at $_log (run default command first)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
_role="$(detect_role)"
|
||||
printf 'log=%s role=%s\n' "$_log" "$_role"
|
||||
report_day_block "$_log" "today" "$_today"
|
||||
|
||||
if [ -n "$_yesterday" ] && grep -qE "^kind=snapshot ts=$_yesterday|^ts=$_yesterday" "$_log" 2>/dev/null; then
|
||||
report_day_block "$_log" "yesterday" "$_yesterday"
|
||||
fi
|
||||
|
||||
printf '\nInterpretation:\n'
|
||||
printf ' wan bad + fe good -> ISP/upstream or route_dev issue\n'
|
||||
printf ' wan + fe bad -> router/LAN/tunnel path\n'
|
||||
printf ' fe bad (pi) + fe ok (fe) -> path to FE, not FE host CPU\n'
|
||||
printf ' high mdev / max vs avg -> jitter (bufferbloat, conntrack, VPN)\n'
|
||||
printf ' conntrack>80%% -> edge table pressure (router)\n'
|
||||
printf ' neigh_fe not REACHABLE -> ARP/L2 flap toward FE\n'
|
||||
printf ' ping good, HTTPS slow -> DNAT/nginx above this probe\n'
|
||||
}
|
||||
|
||||
tail_log() {
|
||||
_n="${1:-24}"
|
||||
_log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
|
||||
[ -r "$_log" ] || {
|
||||
echo "wan-snapshot: no log at $_log" >&2
|
||||
exit 1
|
||||
}
|
||||
grep -E '^kind=snapshot |^ts=' "$_log" 2>/dev/null | tail -n "$_n"
|
||||
}
|
||||
|
||||
tail_alerts() {
|
||||
_n="${1:-40}"
|
||||
_log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
|
||||
[ -r "$_log" ] || {
|
||||
echo "wan-snapshot: no log at $_log" >&2
|
||||
exit 1
|
||||
}
|
||||
grep -E '^kind=alert |^kind=deep ' "$_log" 2>/dev/null | tail -n "$_n"
|
||||
}
|
||||
|
||||
# Guess role label for correlate when log line has no role=.
|
||||
log_role_label() {
|
||||
_path="$1"
|
||||
_line="$2"
|
||||
_role="$(field_value "$_line" "role")"
|
||||
if [ -n "$_role" ] && [ "$_role" != "0" ]; then
|
||||
printf '%s' "$_role"
|
||||
return 0
|
||||
fi
|
||||
case "$_path" in
|
||||
*raspberry*|*rpi*) printf 'pi' ;;
|
||||
*FE_gentoo*|*gentoo*) printf 'fe' ;;
|
||||
*monstro*|*router*) printf 'router' ;;
|
||||
*) printf '%s' "$(basename "$_path" .log)" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Bucket timestamp to 15-min wall-clock slot: YYYY-MM-DDTHH:00|15|30|45
|
||||
slot_key() {
|
||||
_ts="$1"
|
||||
_date="${_ts%%T*}"
|
||||
_rest="${_ts#*T}"
|
||||
_hour="${_rest%%:*}"
|
||||
_min="${_rest#*:}"
|
||||
_min="${_min%%:*}"
|
||||
_min="${_min%%+*}"
|
||||
_min="${_min%%-*}"
|
||||
_bucket=$(( (_min / 15) * 15 ))
|
||||
printf '%sT%s:%02d\n' "$_date" "$_hour" "$_bucket"
|
||||
}
|
||||
|
||||
correlate_logs() {
|
||||
if [ "$#" -lt 1 ]; then
|
||||
echo "wan-snapshot: --correlate needs at least one log file" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
_tmp="$(mktemp "${TMPDIR:-/tmp}/wan-correlate.XXXXXX")"
|
||||
# shellcheck disable=SC2064
|
||||
trap 'rm -f "$_tmp"' EXIT INT HUP
|
||||
|
||||
for _path in "$@"; do
|
||||
[ -r "$_path" ] || {
|
||||
echo "wan-snapshot: unreadable: $_path" >&2
|
||||
continue
|
||||
}
|
||||
_label="$(basename "$_path")"
|
||||
while IFS= read -r _line || [ -n "$_line" ]; do
|
||||
is_snapshot_line "$_line" || continue
|
||||
case "$_line" in
|
||||
kind=snapshot*) _ts="${_line#kind=snapshot ts=}" ;;
|
||||
ts=*) _ts="${_line#ts=}" ;;
|
||||
*) continue ;;
|
||||
esac
|
||||
_ts="${_ts%% *}"
|
||||
_slot="$(slot_key "$_ts")"
|
||||
_role="$(log_role_label "$_path" "$_line")"
|
||||
printf '%s\t%s\t%s\t%s\t%s\n' "$_slot" "$_role" "$_ts" "$_label" "$_line" >> "$_tmp"
|
||||
done < "$_path"
|
||||
done
|
||||
|
||||
printf 'slot\t'
|
||||
_roles="$(awk -F'\t' '{print $2}' "$_tmp" | sort -u | tr '\n' ' ')"
|
||||
for _role in $_roles; do
|
||||
printf '%s_wan\t%s_fe\t%s_be\t' "$_role" "$_role" "$_role"
|
||||
done
|
||||
printf '\n'
|
||||
|
||||
awk -F'\t' '
|
||||
{
|
||||
slot = $1; role = $2; ts = $3; line = $5
|
||||
key = slot SUBSEP role
|
||||
if (!(key in best_ts) || ts > best_ts[key]) {
|
||||
best_ts[key] = ts
|
||||
wan = fe = be = "-"
|
||||
for (i = 1; i <= split(line, a, " "); i++) {
|
||||
if (a[i] ~ /^wan_avg_ms=/) { split(a[i], b, "="); wan = b[2] }
|
||||
if (a[i] ~ /^fe_avg_ms=/) { split(a[i], b, "="); fe = b[2] }
|
||||
if (a[i] ~ /^be_avg_ms=/) { split(a[i], b, "="); be = b[2] }
|
||||
}
|
||||
W[slot, role, "wan"] = wan
|
||||
W[slot, role, "fe"] = fe
|
||||
W[slot, role, "be"] = be
|
||||
slots[slot] = 1
|
||||
roles[role] = 1
|
||||
}
|
||||
}
|
||||
END {
|
||||
nroles = 0
|
||||
for (r in roles) rolelist[++nroles] = r
|
||||
asort(rolelist)
|
||||
nslots = 0
|
||||
for (s in slots) slotlist[++nslots] = s
|
||||
asort(slotlist)
|
||||
for (si = 1; si <= nslots; si++) {
|
||||
s = slotlist[si]
|
||||
printf "%s", s
|
||||
for (ri = 1; ri <= nroles; ri++) {
|
||||
r = rolelist[ri]
|
||||
printf "\t%s\t%s\t%s", W[s, r, "wan"] + 0 ? W[s, r, "wan"] : "-", \
|
||||
W[s, r, "fe"] + 0 ? W[s, r, "fe"] : "-", \
|
||||
W[s, r, "be"] + 0 ? W[s, r, "be"] : "-"
|
||||
}
|
||||
printf "\n"
|
||||
}
|
||||
}
|
||||
' "$_tmp" | while IFS= read -r _row; do
|
||||
printf '%s\n' "$_row"
|
||||
done
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
-h|--help|help)
|
||||
usage
|
||||
;;
|
||||
--report|report)
|
||||
report
|
||||
;;
|
||||
--tail|tail)
|
||||
tail_log "${2:-24}"
|
||||
;;
|
||||
--alerts|alerts)
|
||||
tail_alerts "${2:-40}"
|
||||
;;
|
||||
--correlate|correlate)
|
||||
shift
|
||||
correlate_logs "$@"
|
||||
;;
|
||||
"")
|
||||
snapshot_once
|
||||
;;
|
||||
*)
|
||||
echo "wan-snapshot: unknown command: $1 (try --help)" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
# Backward-compatible wrapper — canonical script is wan-watch.sh in the same directory.
|
||||
D="$(CDPATH= cd -- "$(dirname "$0")" && pwd)"
|
||||
exec "$D/wan-watch.sh" "$@"
|
||||
|
||||
Reference in New Issue
Block a user