ac/android_cast
1
0
Fork 0
mirror of git://f0xx.org/android_cast synced 2026-07-29 08:37:37 +03:00
Code Issues Packages Projects Releases Wiki Activity

bumping

Browse Source 
Net topology: sirene (P2P), spider (MCU), spider net (SFU), pautina (net, mesh), hydra (hybrid), code names
This commit is contained in:
Anton Afanasyeu
2026-06-12 22:13:28 +02:00
parent ad35a60e6c
commit 9acbdbc433
4 changed files with 564 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

234
examples/network/wan-snapshot.sh
View File

@@ -15,24 +15,40 @@
# */15 * * * * root /usr/local/bin/wan-snapshot.sh >>/var/log/wan-quality.cron.log 2>&1
#
# Env overrides:
# WAN_SNAPSHOT_LOG=/var/log/wan-quality.log
# WAN_SNAPSHOT_ROLE=auto|router|fe|pi (default auto from hostname)
# WAN_SNAPSHOT_EXTRAS=1 (0 = ping only, compact v1-style line)
# WAN_WATCH_LOG=/var/log/wan-watch.log (unified: snapshot + alert + deep)
# WAN_SNAPSHOT_LOG=/var/log/wan-quality.log (legacy snapshot copy if different)
# WAN_SNAPSHOT_ROLE=auto|router|fe|pi
# WAN_SNAPSHOT_EXTRAS=1
# WAN_SNAPSHOT_WATCH=1 (0 = disable auto deep-diag on anomaly)
# WAN_SNAPSHOT_LOCK=/var/run/wan-snapshot.lock
# WAN_PING_TARGET=1.1.1.1
# FE_PING_TARGET=10.7.0.10
# BE_PING_TARGET=10.7.16.128 (set to "none" to skip)
# ROUTER_PING_TARGET=none|10.7.6.228 (optional; useful on Pi)
# GW_PING_TARGET=auto|none|<ip> (default auto = default gateway)
# PING_COUNT=10
# PING_DEADLINE_S=15
# WAN_WATCH_COOLDOWN_S=840 (min seconds between deep captures)
# WAN_ALERT_MS=50 FE_ALERT_MS=50 BE_ALERT_MS=50 GW_ALERT_MS=50
# WAN_LOSS_ALERT_PCT=1 WAN_MDEV_ALERT_MS=80 WAN_CONNTRACK_ALERT_PCT=80
# WAN_PING_TARGET=1.1.1.1 FE_PING_TARGET=10.7.0.10
# BE_PING_TARGET=10.7.16.128 ROUTER_PING_TARGET=none GW_PING_TARGET=auto
# PING_COUNT=10 PING_DEADLINE_S=15
#
# Install deep-diag library next to this script:
# cp wan-snapshot.sh wan-deep-diag.sh /usr/local/bin/
#
set -eu
WAN_SNAPSHOT_LOG="${WAN_SNAPSHOT_LOG:-/var/log/wan-quality.log}"
WAN_WATCH_LOG="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
WAN_SNAPSHOT_ROLE="${WAN_SNAPSHOT_ROLE:-auto}"
WAN_SNAPSHOT_EXTRAS="${WAN_SNAPSHOT_EXTRAS:-1}"
WAN_SNAPSHOT_WATCH="${WAN_SNAPSHOT_WATCH:-1}"
WAN_SNAPSHOT_LOCK="${WAN_SNAPSHOT_LOCK:-/var/run/wan-snapshot.lock}"
WAN_WATCH_COOLDOWN_S="${WAN_WATCH_COOLDOWN_S:-840}"
WAN_WATCH_COOLDOWN_FILE="${WAN_WATCH_COOLDOWN_FILE:-/var/run/wan-watch-deep.stamp}"
WAN_ALERT_MS="${WAN_ALERT_MS:-50}"
FE_ALERT_MS="${FE_ALERT_MS:-50}"
BE_ALERT_MS="${BE_ALERT_MS:-50}"
GW_ALERT_MS="${GW_ALERT_MS:-50}"
ROUTER_ALERT_MS="${ROUTER_ALERT_MS:-50}"
WAN_LOSS_ALERT_PCT="${WAN_LOSS_ALERT_PCT:-1}"
WAN_MDEV_ALERT_MS="${WAN_MDEV_ALERT_MS:-80}"
WAN_CONNTRACK_ALERT_PCT="${WAN_CONNTRACK_ALERT_PCT:-80}"
WAN_PING_TARGET="${WAN_PING_TARGET:-1.1.1.1}"
FE_PING_TARGET="${FE_PING_TARGET:-10.7.0.10}"
BE_PING_TARGET="${BE_PING_TARGET-10.7.16.128}"
@@ -41,6 +57,13 @@ GW_PING_TARGET="${GW_PING_TARGET:-auto}"
PING_COUNT="${PING_COUNT:-10}"
PING_DEADLINE_S="${PING_DEADLINE_S:-15}"
_WAN_SNAPSHOT_DIR="$(CDPATH= cd -- "$(dirname "$0")" && pwd)"
# shellcheck source=/dev/null
. "$_WAN_SNAPSHOT_DIR/wan-deep-diag.sh" 2>/dev/null \
|| . /usr/local/bin/wan-deep-diag.sh 2>/dev/null \
|| . /usr/local/lib/wan-deep-diag.sh 2>/dev/null \
|| true
usage() {
sed -n '2,28p' "$0"
printf '\nCommands:\n'
@@ -48,6 +71,7 @@ usage() {
printf ' --report summarize today vs yesterday\n'
printf ' --tail [N] print last N log lines (default 24)\n'
printf ' --correlate FILE… merge logs by 15-min slot (role= or path basename)\n'
printf ' --alerts [N] recent kind=alert + kind=deep lines (default 40)\n'
printf ' --help this text\n'
}
@@ -182,12 +206,149 @@ iface_drops_compact() {
' | sed 's/ $//'
}
# True if line is a snapshot row (legacy ts=… or kind=snapshot).
is_snapshot_line() {
case "$1" in
kind=snapshot*) return 0 ;;
ts=*) return 0 ;;
*) return 1 ;;
esac
}
num_ge() {
_a="$1"
_b="$2"
[ "$_a" != "-1" ] && [ -n "$_a" ] && awk -v a="$_a" -v b="$_b" 'BEGIN { exit (a + 0 >= b + 0) ? 0 : 1 }'
}
watch_cooldown_active() {
[ -r "$WAN_WATCH_COOLDOWN_FILE" ] || return 1
_last="$(cat "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null)" || return 1
_now="$(date +%s 2>/dev/null)" || return 1
[ "$((_now - _last))" -lt "$WAN_WATCH_COOLDOWN_S" ] 2>/dev/null
}
watch_cooldown_stamp() {
date +%s > "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null || true
}
# Returns trigger list (comma-separated) or empty.
detect_anomalies() {
_line="$1"
_role="$2"
_triggers=""
for _pfx in wan fe be gw router; do
_avg="$(field_value "$_line" "${_pfx}_avg_ms")"
_loss="$(field_value "$_line" "${_pfx}_loss")"
_mdev="$(field_value "$_line" "${_pfx}_mdev_ms")"
_max="$(field_value "$_line" "${_pfx}_max_ms")"
_thr="$FE_ALERT_MS"
case "$_pfx" in
wan) _thr="$WAN_ALERT_MS" ;;
fe) _thr="$FE_ALERT_MS" ;;
be) _thr="$BE_ALERT_MS" ;;
gw) _thr="$GW_ALERT_MS" ;;
router) _thr="$ROUTER_ALERT_MS" ;;
esac
if num_ge "$_avg" "$_thr"; then
_triggers="${_triggers}${_triggers:+,}${_pfx}_avg=${_avg}"
fi
if num_ge "$_loss" "$WAN_LOSS_ALERT_PCT"; then
_triggers="${_triggers}${_triggers:+,}${_pfx}_loss=${_loss}"
fi
if num_ge "$_mdev" "$WAN_MDEV_ALERT_MS"; then
_triggers="${_triggers}${_triggers:+,}${_pfx}_mdev=${_mdev}"
fi
if num_ge "$_max" "$((_thr * 3))"; then
_triggers="${_triggers}${_triggers:+,}${_pfx}_max=${_max}"
fi
done
_neigh="$(printf '%s\n' "$_line" | sed -n 's/.*neigh_fe=\([^ ]*\).*/\1/p')"
if [ -n "$_neigh" ] && [ "$_neigh" != "REACHABLE" ] && [ "$_neigh" != "DELAY" ]; then
_triggers="${_triggers}${_triggers:+,}neigh_fe=${_neigh}"
fi
_ct_pct="$(printf '%s\n' "$_line" | sed -n 's/.*(\([0-9]*\)%).*/\1/p' | head -1)"
if [ -n "$_ct_pct" ] && num_ge "$_ct_pct" "$WAN_CONNTRACK_ALERT_PCT"; then
_triggers="${_triggers}${_triggers:+,}conntrack_pct=${_ct_pct}"
fi
# Pi: WAN ok but FE slow -> path anomaly (even if FE just above threshold).
if [ "$_role" = "pi" ]; then
_wan_avg="$(field_value "$_line" "wan_avg_ms")"
_fe_avg="$(field_value "$_line" "fe_avg_ms")"
if num_ge "$_fe_avg" "$FE_ALERT_MS" && ! num_ge "$_wan_avg" "$WAN_ALERT_MS"; then
case "$_triggers" in *fe_avg=*) ;; *)
_triggers="${_triggers}${_triggers:+,}pi_fe_path" ;;
esac
fi
fi
# Router: WAN bad, FE ok -> edge WAN leg (matches monstro xenbr0 pattern).
if [ "$_role" = "router" ]; then
_wan_avg="$(field_value "$_line" "wan_avg_ms")"
_fe_avg="$(field_value "$_line" "fe_avg_ms")"
if num_ge "$_wan_avg" "$WAN_ALERT_MS" && ! num_ge "$_fe_avg" "$FE_ALERT_MS"; then
case "$_triggers" in *wan_avg=*) ;; *)
_triggers="${_triggers}${_triggers:+,}router_wan_path" ;;
esac
fi
fi
printf '%s' "$_triggers"
}
maybe_deep_diag() {
_line="$1"
_ts="$2"
_role="$3"
_hostn="$4"
[ "$WAN_SNAPSHOT_WATCH" = "1" ] || return 0
[ "${WAN_DEEP_DIAG_LOADED:-0}" = "1" ] || {
echo "wan-snapshot: watch enabled but wan-deep-diag.sh not loaded (install alongside wan-snapshot.sh)" >&2
return 0
}
_triggers="$(detect_anomalies "$_line" "$_role")"
[ -n "$_triggers" ] || return 0
if watch_cooldown_active; then
wan_deep_emit alert "$_ts" "$_role" "$_hostn" \
"parent_ts=$_ts triggers=\"$_triggers\" action=skipped_cooldown"
return 0
fi
_gw_target="$GW_PING_TARGET"
if [ "$_gw_target" = "auto" ]; then
_gw_target="$(default_gw)"
fi
_wdev="$(route_dev "$WAN_PING_TARGET")"
_fdev="$(route_dev "$FE_PING_TARGET")"
wan_deep_diag_run "$_ts" "$_role" "$_hostn" "$_triggers" \
"$WAN_PING_TARGET" "$FE_PING_TARGET" "$_gw_target" "$_wdev" "$_fdev"
watch_cooldown_stamp
}
append_watch_log() {
_line="$1"
_dir="$(dirname "$WAN_WATCH_LOG")"
[ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true
printf '%s\n' "$_line" >> "$WAN_WATCH_LOG"
}
acquire_lock() {
command -v flock >/dev/null 2>&1 || return 0
_lock="$WAN_SNAPSHOT_LOCK"
_dir="$(dirname "$_lock")"
if ! mkdir -p "$_dir" 2>/dev/null || ! : >"$_lock" 2>/dev/null; then
_lock="${TMPDIR:-/tmp}/wan-snapshot.$(id -u 2>/dev/null || echo 0).lock"
_fallback="${TMPDIR:-/tmp}/wan-snapshot.$(id -u 2>/dev/null || echo 0).lock"
if [ ! -d "$_dir" ] || [ ! -w "$_dir" ] 2>/dev/null; then
_lock="$_fallback"
elif ! ( : >>"$_lock" ) 2>/dev/null; then
_lock="$_fallback"
fi
exec 9>"$_lock" 2>/dev/null || return 0
flock -n 9 || {
@@ -204,7 +365,7 @@ snapshot_once() {
_role="$(detect_role)"
_host="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)"
_line="ts=$_ts ver=2 role=$_role host=$_host"
_line="kind=snapshot ts=$_ts ver=2 role=$_role host=$_host"
_line="$(append_ping_fields "$_line" "wan" "$WAN_PING_TARGET")"
_line="$(append_ping_fields "$_line" "fe" "$FE_PING_TARGET")"
@@ -241,9 +402,15 @@ snapshot_once() {
[ -n "$_drops" ] && _line="$_line drops=\"$_drops\""
fi
_dir="$(dirname "$WAN_SNAPSHOT_LOG")"
[ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true
printf '%s\n' "$_line" >> "$WAN_SNAPSHOT_LOG"
append_watch_log "$_line"
if [ "$WAN_SNAPSHOT_LOG" != "$WAN_WATCH_LOG" ]; then
_dir="$(dirname "$WAN_SNAPSHOT_LOG")"
[ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true
printf '%s\n' "$_line" >> "$WAN_SNAPSHOT_LOG"
fi
maybe_deep_diag "$_line" "$_ts" "$_role" "$_host"
printf '%s\n' "$_line"
}
@@ -271,7 +438,7 @@ day_stats() {
_field="$3"
[ -r "$_log" ] || return 0
awk -v d="$_date" -v f="$_field" '
$0 ~ "^ts=" d {
($0 ~ "^kind=snapshot ts=" d || ($0 ~ "^ts=" d && $0 !~ / kind=/)) {
for (i = 1; i <= NF; i++) {
if ($i ~ "^" f "=") {
split($i, a, "=")
@@ -302,7 +469,7 @@ day_max_loss() {
_field="$3"
[ -r "$_log" ] || return 0
awk -v d="$_date" -v f="$_field" '
$0 ~ "^ts=" d {
($0 ~ "^kind=snapshot ts=" d || ($0 ~ "^ts=" d && $0 !~ / kind=/)) {
for (i = 1; i <= NF; i++) {
if ($i ~ "^" f "=") {
split($i, a, "=")
@@ -323,7 +490,7 @@ report_day_block() {
_log="$1"
_label="$2"
_date="$3"
_samples="$(grep -c "^ts=$_date" "$_log" 2>/dev/null || echo 0)"
_samples="$(grep -cE "^kind=snapshot ts=$_date|^ts=$_date" "$_log" 2>/dev/null || echo 0)"
printf '%s=%s samples=%s\n' "$_label" "$_date" "$_samples"
for _pfx in wan fe be router gw; do
_stats="$(day_stats "$_log" "$_date" "${_pfx}_avg_ms")"
@@ -344,7 +511,7 @@ report_day_block() {
}
report() {
_log="$WAN_SNAPSHOT_LOG"
_log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
_today="$(date '+%Y-%m-%d')"
_yesterday="$(date -d 'yesterday' '+%Y-%m-%d' 2>/dev/null || date -v-1d '+%Y-%m-%d' 2>/dev/null || echo '')"
@@ -357,7 +524,7 @@ report() {
printf 'log=%s role=%s\n' "$_log" "$_role"
report_day_block "$_log" "today" "$_today"
if [ -n "$_yesterday" ] && grep -q "^ts=$_yesterday" "$_log" 2>/dev/null; then
if [ -n "$_yesterday" ] && grep -qE "^kind=snapshot ts=$_yesterday|^ts=$_yesterday" "$_log" 2>/dev/null; then
report_day_block "$_log" "yesterday" "$_yesterday"
fi
@@ -373,11 +540,22 @@ report() {
tail_log() {
_n="${1:-24}"
[ -r "$WAN_SNAPSHOT_LOG" ] || {
echo "wan-snapshot: no log at $WAN_SNAPSHOT_LOG" >&2
_log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
[ -r "$_log" ] || {
echo "wan-snapshot: no log at $_log" >&2
exit 1
}
tail -n "$_n" "$WAN_SNAPSHOT_LOG"
grep -E '^kind=snapshot |^ts=' "$_log" 2>/dev/null | tail -n "$_n"
}
tail_alerts() {
_n="${1:-40}"
_log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
[ -r "$_log" ] || {
echo "wan-snapshot: no log at $_log" >&2
exit 1
}
grep -E '^kind=alert |^kind=deep ' "$_log" 2>/dev/null | tail -n "$_n"
}
# Guess role label for correlate when log line has no role=.
@@ -428,11 +606,12 @@ correlate_logs() {
}
_label="$(basename "$_path")"
while IFS= read -r _line || [ -n "$_line" ]; do
is_snapshot_line "$_line" || continue
case "$_line" in
ts=*) ;;
kind=snapshot*) _ts="${_line#kind=snapshot ts=}" ;;
ts=*) _ts="${_line#ts=}" ;;
*) continue ;;
esac
_ts="${_line#ts=}"
_ts="${_ts%% *}"
_slot="$(slot_key "$_ts")"
_role="$(log_role_label "$_path" "$_line")"
@@ -500,6 +679,9 @@ case "${1:-}" in
--tail|tail)
tail_log "${2:-24}"
;;
--alerts|alerts)
tail_alerts "${2:-40}"
;;
--correlate|correlate)
shift
correlate_logs "$@"