be sync and infra sync

2026-07-29 04:18:09 +03:00 · 2026-06-11 10:28:09 +02:00
parent cb6187b014
commit 57f0e8b465
3 changed files with 295 additions and 1 deletions
--- a/docs/INFRA.md
+++ b/docs/INFRA.md
@@ -232,5 +232,5 @@ Expect **200** (or **302** → crashes only if you intentionally redirect root).

 | Date | Note |
 |------|------|
-| 2026-06-04 | Edge router WAN probe: `examples/network/wan-snapshot.sh` (world ↔ 134.17.26.161 vs 10.7.0.10) |
+| 2026-06-04 | Edge probes: `examples/network/wan-snapshot.sh`, `monstro-netdiag.sh` (conntrack/iptables/CPU) |
 | 2026-05-21 | Documented BE-only hub on port 80; 8089 = Janus only; FE `apps` vs `artc0` split |
--- a/examples/network/README.md
+++ b/examples/network/README.md
@@ -59,3 +59,61 @@ mtr -rwzc 50 134.17.26.161
 ```

 That catches path issues ping from the router cannot see.
+
+## `monstro-netdiag.sh` (bottleneck + iptables + processes)
+
+Run on the **Debian edge router** (`10.7.6.228`, same host as `wan-snapshot.sh`) when infra feels stuck.
+
+```bash
+sudo cp examples/network/monstro-netdiag.sh /usr/local/bin/
+sudo chmod +x /usr/local/bin/monstro-netdiag.sh
+```
+
+**Cron** (pair with wan-snapshot):
+
+```cron
+*/15 * * * * root /usr/local/bin/wan-snapshot.sh >>/var/log/wan-quality.cron.log 2>&1
+*/15 * * * * root /usr/local/bin/monstro-netdiag.sh --snapshot >>/var/log/monstro-netdiag.cron.log 2>&1
+```
+
+**When sluggish** (run immediately, compare to a “good” capture):
+
+```bash
+sudo monstro-netdiag.sh --full          # ping + conntrack + iptables -v + top CPU
+sudo monstro-netdiag.sh --watch 30      # stream one-liners every 30s
+monstro-netdiag.sh --tail 48
+```
+
+One-line log (`/var/log/monstro-netdiag.log`) includes: `load1`, `cpu_busy`, `conntrack=cur/max (%)`, `top=process`, `fe_avg_ms`, `wan_avg_ms`, iface drops.
+
+### Why intermittent (excluding ISP)?
+
+Your data showed **WAN ~9 ms** but **FE ~230 ms** — classic **intra-LAN** flapping:
+
+| Cause | What to look for in `--full` |
+|-------|------------------------------|
+| **conntrack table full** | `conntrack` >80%, new flows stall |
+| **Router CPU / softirq** | high `cpu_busy`, `softnet dropped` |
+| **iptables rule cost** | FORWARD/NAT rules with huge packet counts |
+| **FE/BE VM host busy** | FE ping high from router, low load on router |
+| **ARP/route flap** | `ip neigh` INCOMPLETE/FAILED for 10.7.0.10 |
+| **Bufferbloat on LAN** | rising `fe_avg_ms` under load, 0% loss |
+
+Good vs bad: save one `--full` output on a rocketship evening and diff when stuck.
+
+### Safe checks on monstro (read-only)
+
+```bash
+# conntrack pressure
+cat /proc/sys/net/netfilter/nf_conntrack_count
+cat /proc/sys/net/netfilter/nf_conntrack_max
+
+# hottest iptables rules (no changes)
+iptables -L FORWARD -n -v --line-numbers | sort -k1 -rn | head
+iptables -t nat -L PREROUTING -n -v --line-numbers | head -20
+
+# who eats CPU during slowness
+ps -eo pcpu,pmem,comm --sort=-pcpu | head
+```
+
+Do **not** change iptables/NAT on monstro without a rollback plan (per infra policy).
--- a/examples/network/monstro-netdiag.sh
+++ b/examples/network/monstro-netdiag.sh
@@ -0,0 +1,236 @@
+#!/bin/sh
+#
+# Edge-router bottleneck snapshot (monstro / Debian 10.7.6.228).
+# Run when LAN feels slow: compares WAN vs FE/BE ping with host health signals.
+#
+# Install on router (same host as wan-snapshot.sh):
+#   sudo cp monstro-netdiag.sh /usr/local/bin/monstro-netdiag.sh
+#   sudo chmod +x /usr/local/bin/monstro-netdiag.sh
+#
+# Cron (every 15 min, one-line log):
+#   */15 * * * * root /usr/local/bin/monstro-netdiag.sh --snapshot
+#
+# When sluggish — full report:
+#   sudo monstro-netdiag.sh --full
+#   sudo monstro-netdiag.sh --watch 30
+#
+set -eu
+
+MONSTRO_LOG="${MONSTRO_LOG:-/var/log/monstro-netdiag.log}"
+WAN_TARGET="${WAN_TARGET:-1.1.1.1}"
+FE_TARGET="${FE_TARGET:-10.7.0.10}"
+BE_TARGET="${BE_TARGET:-10.7.16.128}"
+PING_COUNT="${PING_COUNT:-5}"
+PING_DEADLINE_S="${PING_DEADLINE_S:-8}"
+
+log() { printf '%s\n' "$*"; }
+section() { printf '\n=== %s ===\n' "$*"; }
+
+ping_avg_ms() {
+  _host="$1"
+  _out="$(ping -c "$PING_COUNT" -w "$PING_DEADLINE_S" "$_host" 2>/dev/null)" || true
+  printf '%s\n' "$_out" | awk -F'[=/ ]' '/min\/avg\/max/ { print $8; exit }'
+}
+
+ping_loss_pct() {
+  _host="$1"
+  _out="$(ping -c "$PING_COUNT" -w "$PING_DEADLINE_S" "$_host" 2>/dev/null)" || true
+  printf '%s\n' "$_out" | awk '/packet loss/ {
+    for (i = 1; i <= NF; i++) if ($i ~ /%$/) { gsub(/%/, "", $i); print $i; exit }
+  }'
+}
+
+load_1m() {
+  awk '{print $1}' /proc/loadavg 2>/dev/null || echo "?"
+}
+
+cpu_busy_pct() {
+  awk '/^cpu / {
+    idle=$5+$6; total=0; for(i=2;i<=NF;i++) total+=$i
+    if (total>0) printf "%.0f", 100*(total-idle)/total
+  }' /proc/stat 2>/dev/null || echo "?"
+}
+
+conntrack_line() {
+  _cur="$(cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null)" || _cur=""
+  _max="$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null)" || _max=""
+  if [ -n "$_cur" ] && [ -n "$_max" ] && [ "$_max" -gt 0 ] 2>/dev/null; then
+    _pct=$((_cur * 100 / _max))
+    printf 'conntrack=%s/%s (%s%%)' "$_cur" "$_max" "$_pct"
+  else
+    printf 'conntrack=n/a'
+  fi
+}
+
+top_cpu_process() {
+  ps -eo pcpu,pmem,comm --sort=-pcpu 2>/dev/null | awk 'NR==2 {printf "%s cpu=%s%% mem=%s%%", $3, $1, $2; exit}'
+  [ $? -eq 0 ] || ps aux 2>/dev/null | awk 'NR>1 {print $3,$4,$11; exit}' | awk '{printf "%s cpu=%s%% mem=%s%%", $3, $1, $2}'
+}
+
+iface_drops_line() {
+  # Sum RX/TX drop+error on non-loopback interfaces.
+  ip -s link 2>/dev/null | awk '
+    /^[0-9]+:/ {
+      if (iface != "" && iface !~ /^lo/) {
+        printf "%s:rx_drop=%d tx_drop=%d rx_err=%d tx_err=%d ", iface, rx_drop, tx_drop, rx_err, tx_err
+      }
+      iface=$2; gsub(":", "", iface); rx_drop=tx_drop=rx_err=tx_err=0
+    }
+    /RX:/{ getline; rx_drop=$4; rx_err=$3 }
+    /TX:/{ getline; tx_drop=$4; tx_err=$3 }
+    END {
+      if (iface != "" && iface !~ /^lo/)
+        printf "%s:rx_drop=%d tx_drop=%d rx_err=%d tx_err=%d", iface, rx_drop, tx_drop, rx_err, tx_err
+    }
+  ' | tr ' ' '\n' | grep -v '=0$' | tr '\n' ' ' | sed 's/ $//'
+}
+
+snapshot_once() {
+  _ts="$(date -Iseconds 2>/dev/null || date '+%Y-%m-%dT%H:%M:%S%z')"
+  _wan_avg="$(ping_avg_ms "$WAN_TARGET")"; [ -n "$_wan_avg" ] || _wan_avg="-1"
+  _fe_avg="$(ping_avg_ms "$FE_TARGET")"; [ -n "$_fe_avg" ] || _fe_avg="-1"
+  _be_avg="$(ping_avg_ms "$BE_TARGET")"; [ -n "$_be_avg" ] || _be_avg="-1"
+  _wan_loss="$(ping_loss_pct "$WAN_TARGET")"; [ -n "$_wan_loss" ] || _wan_loss="-1"
+  _fe_loss="$(ping_loss_pct "$FE_TARGET")"; [ -n "$_fe_loss" ] || _fe_loss="-1"
+  _load="$(load_1m)"
+  _cpu="$(cpu_busy_pct)"
+  _ct="$(conntrack_line)"
+  _top="$(top_cpu_process)"
+  _drops="$(iface_drops_line)"
+  _line="ts=$_ts load1=$_load cpu_busy=${_cpu}% $_ct top=\"$_top\" wan_avg_ms=$_wan_avg wan_loss=${_wan_loss}% fe_avg_ms=$_fe_avg fe_loss=${_fe_loss}% be_avg_ms=$_be_avg"
+  [ -n "$_drops" ] && _line="$_line drops=\"$_drops\""
+  _dir="$(dirname "$MONSTRO_LOG")"
+  [ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true
+  printf '%s\n' "$_line" >> "$MONSTRO_LOG"
+  printf '%s\n' "$_line"
+}
+
+print_ping_block() {
+  _label="$1"
+  _host="$2"
+  section "ping $_label ($_host)"
+  ping -c "$PING_COUNT" -w "$PING_DEADLINE_S" "$_host" 2>/dev/null || warn "ping failed"
+}
+
+print_host_health() {
+  section "host $(hostname -s 2>/dev/null || echo router)"
+  log "uptime: $(uptime 2>/dev/null || true)"
+  log "load: $(cat /proc/loadavg 2>/dev/null || echo n/a)"
+  log "mem: $(free -h 2>/dev/null | awk '/^Mem:/ {print $3 "/" $2}' || true)"
+  log "$(conntrack_line)"
+  log "top cpu: $(top_cpu_process)"
+  _drops="$(iface_drops_line)"
+  [ -n "$_drops" ] && log "iface drops/errors: $_drops"
+}
+
+print_iptables_summary() {
+  section "iptables (packet counts — hot rules first)"
+  if ! command -v iptables >/dev/null 2>&1; then
+    log "iptables not installed"
+    return 0
+  fi
+  for _tbl in filter nat mangle; do
+    log "--- table $_tbl FORWARD (top 12 by packets) ---"
+    iptables -t "$_tbl" -L FORWARD -n -v --line-numbers 2>/dev/null \
+      | awk 'NR==1 || ($1+0)>0' | head -n 13 || true
+    log "--- table $_tbl PREROUTING (top 8) ---"
+    iptables -t "$_tbl" -L PREROUTING -n -v --line-numbers 2>/dev/null \
+      | awk 'NR==1 || ($1+0)>0' | head -n 9 || true
+  done
+  log "--- NAT POSTROUTING (MASQUERADE/SNAT) ---"
+  iptables -t nat -L POSTROUTING -n -v --line-numbers 2>/dev/null | head -n 15 || true
+}
+
+print_processes() {
+  section "top processes (cpu)"
+  ps -eo pcpu,pmem,pid,user,comm,args --sort=-pcpu 2>/dev/null | head -n 12 \
+    || ps aux --sort=-%cpu 2>/dev/null | head -n 12
+}
+
+print_routing() {
+  section "routes to FE/BE"
+  ip route get "$FE_TARGET" 2>/dev/null || true
+  ip route get "$BE_TARGET" 2>/dev/null || true
+  section "arp (FE)"
+  ip neigh show "$FE_TARGET" 2>/dev/null || arp -n "$FE_TARGET" 2>/dev/null || true
+}
+
+print_softnet() {
+  section "softnet drops (per-cpu, col3=processed col2=dropped)"
+  if [ -r /proc/net/softnet_stat ]; then
+    awk '{ if ($2+0 > 0) printf "cpu%02d dropped=%s processed=%s\n", NR-1, $2, $1 }' /proc/net/softnet_stat
+  fi
+}
+
+print_wg() {
+  command -v wg >/dev/null 2>&1 || return 0
+  section "wireguard"
+  wg show 2>/dev/null || true
+}
+
+interpret() {
+  section "interpretation"
+  _fe="$(ping_avg_ms "$FE_TARGET")"
+  _wan="$(ping_avg_ms "$WAN_TARGET")"
+  _fe_i="${_fe%%.*}"; _wan_i="${_wan%%.*}"
+  case "$_fe" in
+    ''|?*) log "fe ping failed — routing/ARP/firewall" ;;
+    *)
+      if [ "${_fe_i:-999}" -gt 50 ] 2>/dev/null && [ "${_wan_i:-0}" -lt 30 ] 2>/dev/null; then
+        log "LAN bottleneck: WAN ok (~${_wan}ms) but FE slow (~${_fe}ms)"
+        log "  check: conntrack %%, iptables FORWARD/NAT, router CPU, FE VM load"
+        log "  intermittent: conntrack fill, NAT spike, ARP refresh, hypervisor scheduling"
+      elif [ "${_fe_i:-0}" -gt 50 ] 2>/dev/null && [ "${_wan_i:-0}" -gt 50 ] 2>/dev/null; then
+        log "WAN + LAN both slow — router overload and/or upstream"
+      else
+        log "ping ok from this host (wan ~${_wan}ms fe ~${_fe}ms)"
+        log "  apps still slow? DNAT/nginx on FE or BE services (above L3 ping)"
+      fi
+      ;;
+  esac
+}
+
+full_report() {
+  log "monstro-netdiag full @ $(date -Iseconds 2>/dev/null || date)"
+  print_host_health
+  print_ping_block "WAN" "$WAN_TARGET"
+  print_ping_block "FE" "$FE_TARGET"
+  print_ping_block "BE" "$BE_TARGET"
+  print_routing
+  print_softnet
+  print_iptables_summary
+  print_processes
+  print_wg
+  interpret
+}
+
+watch_loop() {
+  _sec="${1:-30}"
+  log "watch every ${_sec}s (Ctrl-C to stop) — log: $MONSTRO_LOG"
+  while true; do
+    snapshot_once
+    sleep "$_sec"
+  done
+}
+
+usage() {
+  sed -n '2,18p' "$0"
+  printf '\nCommands:\n'
+  printf '  --snapshot     one line -> %s (cron)\n' "$MONSTRO_LOG"
+  printf '  --full         verbose report when sluggish\n'
+  printf '  --watch [SEC]  repeat snapshot (default 30s)\n'
+  printf '  --tail [N]     last N log lines\n'
+}
+
+case "${1:-}" in
+  -h|--help|help) usage ;;
+  --snapshot|snapshot) snapshot_once ;;
+  --full|full) full_report ;;
+  --watch|watch) watch_loop "${2:-30}" ;;
+  --tail|tail)
+    _n="${2:-20}"
+    [ -r "$MONSTRO_LOG" ] && tail -n "$_n" "$MONSTRO_LOG" || log "no log yet"
+    ;;
+  "") full_report ;;
+  *) log "unknown: $1 (try --help)"; exit 1 ;;
+esac