diff --git a/docs/INFRA.md b/docs/INFRA.md index a3f963a..a774bcf 100644 --- a/docs/INFRA.md +++ b/docs/INFRA.md @@ -232,5 +232,5 @@ Expect **200** (or **302** → crashes only if you intentionally redirect root). | Date | Note | |------|------| -| 2026-06-04 | Edge router WAN probe: `examples/network/wan-snapshot.sh` (world ↔ 134.17.26.161 vs 10.7.0.10) | +| 2026-06-04 | Edge probes: `examples/network/wan-snapshot.sh`, `monstro-netdiag.sh` (conntrack/iptables/CPU) | | 2026-05-21 | Documented BE-only hub on port 80; 8089 = Janus only; FE `apps` vs `artc0` split | diff --git a/examples/network/README.md b/examples/network/README.md index 07f451a..a690d50 100644 --- a/examples/network/README.md +++ b/examples/network/README.md @@ -59,3 +59,61 @@ mtr -rwzc 50 134.17.26.161 ``` That catches path issues ping from the router cannot see. + +## `monstro-netdiag.sh` (bottleneck + iptables + processes) + +Run on the **Debian edge router** (`10.7.6.228`, same host as `wan-snapshot.sh`) when infra feels stuck. + +```bash +sudo cp examples/network/monstro-netdiag.sh /usr/local/bin/ +sudo chmod +x /usr/local/bin/monstro-netdiag.sh +``` + +**Cron** (pair with wan-snapshot): + +```cron +*/15 * * * * root /usr/local/bin/wan-snapshot.sh >>/var/log/wan-quality.cron.log 2>&1 +*/15 * * * * root /usr/local/bin/monstro-netdiag.sh --snapshot >>/var/log/monstro-netdiag.cron.log 2>&1 +``` + +**When sluggish** (run immediately, compare to a “good” capture): + +```bash +sudo monstro-netdiag.sh --full # ping + conntrack + iptables -v + top CPU +sudo monstro-netdiag.sh --watch 30 # stream one-liners every 30s +monstro-netdiag.sh --tail 48 +``` + +One-line log (`/var/log/monstro-netdiag.log`) includes: `load1`, `cpu_busy`, `conntrack=cur/max (%)`, `top=process`, `fe_avg_ms`, `wan_avg_ms`, iface drops. + +### Why intermittent (excluding ISP)? + +Your data showed **WAN ~9 ms** but **FE ~230 ms** — classic **intra-LAN** flapping: + +| Cause | What to look for in `--full` | +|-------|------------------------------| +| **conntrack table full** | `conntrack` >80%, new flows stall | +| **Router CPU / softirq** | high `cpu_busy`, `softnet dropped` | +| **iptables rule cost** | FORWARD/NAT rules with huge packet counts | +| **FE/BE VM host busy** | FE ping high from router, low load on router | +| **ARP/route flap** | `ip neigh` INCOMPLETE/FAILED for 10.7.0.10 | +| **Bufferbloat on LAN** | rising `fe_avg_ms` under load, 0% loss | + +Good vs bad: save one `--full` output on a rocketship evening and diff when stuck. + +### Safe checks on monstro (read-only) + +```bash +# conntrack pressure +cat /proc/sys/net/netfilter/nf_conntrack_count +cat /proc/sys/net/netfilter/nf_conntrack_max + +# hottest iptables rules (no changes) +iptables -L FORWARD -n -v --line-numbers | sort -k1 -rn | head +iptables -t nat -L PREROUTING -n -v --line-numbers | head -20 + +# who eats CPU during slowness +ps -eo pcpu,pmem,comm --sort=-pcpu | head +``` + +Do **not** change iptables/NAT on monstro without a rollback plan (per infra policy). diff --git a/examples/network/monstro-netdiag.sh b/examples/network/monstro-netdiag.sh new file mode 100755 index 0000000..75f71a9 --- /dev/null +++ b/examples/network/monstro-netdiag.sh @@ -0,0 +1,236 @@ +#!/bin/sh +# +# Edge-router bottleneck snapshot (monstro / Debian 10.7.6.228). +# Run when LAN feels slow: compares WAN vs FE/BE ping with host health signals. +# +# Install on router (same host as wan-snapshot.sh): +# sudo cp monstro-netdiag.sh /usr/local/bin/monstro-netdiag.sh +# sudo chmod +x /usr/local/bin/monstro-netdiag.sh +# +# Cron (every 15 min, one-line log): +# */15 * * * * root /usr/local/bin/monstro-netdiag.sh --snapshot +# +# When sluggish — full report: +# sudo monstro-netdiag.sh --full +# sudo monstro-netdiag.sh --watch 30 +# +set -eu + +MONSTRO_LOG="${MONSTRO_LOG:-/var/log/monstro-netdiag.log}" +WAN_TARGET="${WAN_TARGET:-1.1.1.1}" +FE_TARGET="${FE_TARGET:-10.7.0.10}" +BE_TARGET="${BE_TARGET:-10.7.16.128}" +PING_COUNT="${PING_COUNT:-5}" +PING_DEADLINE_S="${PING_DEADLINE_S:-8}" + +log() { printf '%s\n' "$*"; } +section() { printf '\n=== %s ===\n' "$*"; } + +ping_avg_ms() { + _host="$1" + _out="$(ping -c "$PING_COUNT" -w "$PING_DEADLINE_S" "$_host" 2>/dev/null)" || true + printf '%s\n' "$_out" | awk -F'[=/ ]' '/min\/avg\/max/ { print $8; exit }' +} + +ping_loss_pct() { + _host="$1" + _out="$(ping -c "$PING_COUNT" -w "$PING_DEADLINE_S" "$_host" 2>/dev/null)" || true + printf '%s\n' "$_out" | awk '/packet loss/ { + for (i = 1; i <= NF; i++) if ($i ~ /%$/) { gsub(/%/, "", $i); print $i; exit } + }' +} + +load_1m() { + awk '{print $1}' /proc/loadavg 2>/dev/null || echo "?" +} + +cpu_busy_pct() { + awk '/^cpu / { + idle=$5+$6; total=0; for(i=2;i<=NF;i++) total+=$i + if (total>0) printf "%.0f", 100*(total-idle)/total + }' /proc/stat 2>/dev/null || echo "?" +} + +conntrack_line() { + _cur="$(cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null)" || _cur="" + _max="$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null)" || _max="" + if [ -n "$_cur" ] && [ -n "$_max" ] && [ "$_max" -gt 0 ] 2>/dev/null; then + _pct=$((_cur * 100 / _max)) + printf 'conntrack=%s/%s (%s%%)' "$_cur" "$_max" "$_pct" + else + printf 'conntrack=n/a' + fi +} + +top_cpu_process() { + ps -eo pcpu,pmem,comm --sort=-pcpu 2>/dev/null | awk 'NR==2 {printf "%s cpu=%s%% mem=%s%%", $3, $1, $2; exit}' + [ $? -eq 0 ] || ps aux 2>/dev/null | awk 'NR>1 {print $3,$4,$11; exit}' | awk '{printf "%s cpu=%s%% mem=%s%%", $3, $1, $2}' +} + +iface_drops_line() { + # Sum RX/TX drop+error on non-loopback interfaces. + ip -s link 2>/dev/null | awk ' + /^[0-9]+:/ { + if (iface != "" && iface !~ /^lo/) { + printf "%s:rx_drop=%d tx_drop=%d rx_err=%d tx_err=%d ", iface, rx_drop, tx_drop, rx_err, tx_err + } + iface=$2; gsub(":", "", iface); rx_drop=tx_drop=rx_err=tx_err=0 + } + /RX:/{ getline; rx_drop=$4; rx_err=$3 } + /TX:/{ getline; tx_drop=$4; tx_err=$3 } + END { + if (iface != "" && iface !~ /^lo/) + printf "%s:rx_drop=%d tx_drop=%d rx_err=%d tx_err=%d", iface, rx_drop, tx_drop, rx_err, tx_err + } + ' | tr ' ' '\n' | grep -v '=0$' | tr '\n' ' ' | sed 's/ $//' +} + +snapshot_once() { + _ts="$(date -Iseconds 2>/dev/null || date '+%Y-%m-%dT%H:%M:%S%z')" + _wan_avg="$(ping_avg_ms "$WAN_TARGET")"; [ -n "$_wan_avg" ] || _wan_avg="-1" + _fe_avg="$(ping_avg_ms "$FE_TARGET")"; [ -n "$_fe_avg" ] || _fe_avg="-1" + _be_avg="$(ping_avg_ms "$BE_TARGET")"; [ -n "$_be_avg" ] || _be_avg="-1" + _wan_loss="$(ping_loss_pct "$WAN_TARGET")"; [ -n "$_wan_loss" ] || _wan_loss="-1" + _fe_loss="$(ping_loss_pct "$FE_TARGET")"; [ -n "$_fe_loss" ] || _fe_loss="-1" + _load="$(load_1m)" + _cpu="$(cpu_busy_pct)" + _ct="$(conntrack_line)" + _top="$(top_cpu_process)" + _drops="$(iface_drops_line)" + _line="ts=$_ts load1=$_load cpu_busy=${_cpu}% $_ct top=\"$_top\" wan_avg_ms=$_wan_avg wan_loss=${_wan_loss}% fe_avg_ms=$_fe_avg fe_loss=${_fe_loss}% be_avg_ms=$_be_avg" + [ -n "$_drops" ] && _line="$_line drops=\"$_drops\"" + _dir="$(dirname "$MONSTRO_LOG")" + [ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true + printf '%s\n' "$_line" >> "$MONSTRO_LOG" + printf '%s\n' "$_line" +} + +print_ping_block() { + _label="$1" + _host="$2" + section "ping $_label ($_host)" + ping -c "$PING_COUNT" -w "$PING_DEADLINE_S" "$_host" 2>/dev/null || warn "ping failed" +} + +print_host_health() { + section "host $(hostname -s 2>/dev/null || echo router)" + log "uptime: $(uptime 2>/dev/null || true)" + log "load: $(cat /proc/loadavg 2>/dev/null || echo n/a)" + log "mem: $(free -h 2>/dev/null | awk '/^Mem:/ {print $3 "/" $2}' || true)" + log "$(conntrack_line)" + log "top cpu: $(top_cpu_process)" + _drops="$(iface_drops_line)" + [ -n "$_drops" ] && log "iface drops/errors: $_drops" +} + +print_iptables_summary() { + section "iptables (packet counts — hot rules first)" + if ! command -v iptables >/dev/null 2>&1; then + log "iptables not installed" + return 0 + fi + for _tbl in filter nat mangle; do + log "--- table $_tbl FORWARD (top 12 by packets) ---" + iptables -t "$_tbl" -L FORWARD -n -v --line-numbers 2>/dev/null \ + | awk 'NR==1 || ($1+0)>0' | head -n 13 || true + log "--- table $_tbl PREROUTING (top 8) ---" + iptables -t "$_tbl" -L PREROUTING -n -v --line-numbers 2>/dev/null \ + | awk 'NR==1 || ($1+0)>0' | head -n 9 || true + done + log "--- NAT POSTROUTING (MASQUERADE/SNAT) ---" + iptables -t nat -L POSTROUTING -n -v --line-numbers 2>/dev/null | head -n 15 || true +} + +print_processes() { + section "top processes (cpu)" + ps -eo pcpu,pmem,pid,user,comm,args --sort=-pcpu 2>/dev/null | head -n 12 \ + || ps aux --sort=-%cpu 2>/dev/null | head -n 12 +} + +print_routing() { + section "routes to FE/BE" + ip route get "$FE_TARGET" 2>/dev/null || true + ip route get "$BE_TARGET" 2>/dev/null || true + section "arp (FE)" + ip neigh show "$FE_TARGET" 2>/dev/null || arp -n "$FE_TARGET" 2>/dev/null || true +} + +print_softnet() { + section "softnet drops (per-cpu, col3=processed col2=dropped)" + if [ -r /proc/net/softnet_stat ]; then + awk '{ if ($2+0 > 0) printf "cpu%02d dropped=%s processed=%s\n", NR-1, $2, $1 }' /proc/net/softnet_stat + fi +} + +print_wg() { + command -v wg >/dev/null 2>&1 || return 0 + section "wireguard" + wg show 2>/dev/null || true +} + +interpret() { + section "interpretation" + _fe="$(ping_avg_ms "$FE_TARGET")" + _wan="$(ping_avg_ms "$WAN_TARGET")" + _fe_i="${_fe%%.*}"; _wan_i="${_wan%%.*}" + case "$_fe" in + ''|?*) log "fe ping failed — routing/ARP/firewall" ;; + *) + if [ "${_fe_i:-999}" -gt 50 ] 2>/dev/null && [ "${_wan_i:-0}" -lt 30 ] 2>/dev/null; then + log "LAN bottleneck: WAN ok (~${_wan}ms) but FE slow (~${_fe}ms)" + log " check: conntrack %%, iptables FORWARD/NAT, router CPU, FE VM load" + log " intermittent: conntrack fill, NAT spike, ARP refresh, hypervisor scheduling" + elif [ "${_fe_i:-0}" -gt 50 ] 2>/dev/null && [ "${_wan_i:-0}" -gt 50 ] 2>/dev/null; then + log "WAN + LAN both slow — router overload and/or upstream" + else + log "ping ok from this host (wan ~${_wan}ms fe ~${_fe}ms)" + log " apps still slow? DNAT/nginx on FE or BE services (above L3 ping)" + fi + ;; + esac +} + +full_report() { + log "monstro-netdiag full @ $(date -Iseconds 2>/dev/null || date)" + print_host_health + print_ping_block "WAN" "$WAN_TARGET" + print_ping_block "FE" "$FE_TARGET" + print_ping_block "BE" "$BE_TARGET" + print_routing + print_softnet + print_iptables_summary + print_processes + print_wg + interpret +} + +watch_loop() { + _sec="${1:-30}" + log "watch every ${_sec}s (Ctrl-C to stop) — log: $MONSTRO_LOG" + while true; do + snapshot_once + sleep "$_sec" + done +} + +usage() { + sed -n '2,18p' "$0" + printf '\nCommands:\n' + printf ' --snapshot one line -> %s (cron)\n' "$MONSTRO_LOG" + printf ' --full verbose report when sluggish\n' + printf ' --watch [SEC] repeat snapshot (default 30s)\n' + printf ' --tail [N] last N log lines\n' +} + +case "${1:-}" in + -h|--help|help) usage ;; + --snapshot|snapshot) snapshot_once ;; + --full|full) full_report ;; + --watch|watch) watch_loop "${2:-30}" ;; + --tail|tail) + _n="${2:-20}" + [ -r "$MONSTRO_LOG" ] && tail -n "$_n" "$MONSTRO_LOG" || log "no log yet" + ;; + "") full_report ;; + *) log "unknown: $1 (try --help)"; exit 1 ;; +esac