android_cast/examples/network/monstro-netdiag.sh

#!/bin/sh
#
# Edge-router bottleneck snapshot (monstro / Debian 10.7.6.228).
# Run when LAN feels slow: compares WAN vs FE/BE ping with host health signals.
#
# Install on router (same host as wan-snapshot.sh):
#   sudo cp monstro-netdiag.sh /usr/local/bin/monstro-netdiag.sh
#   sudo chmod +x /usr/local/bin/monstro-netdiag.sh
#
# Cron (every 15 min, one-line log):
#   */15 * * * * root /usr/local/bin/monstro-netdiag.sh --snapshot
#
# When sluggish — full report:
#   sudo monstro-netdiag.sh --full
#   sudo monstro-netdiag.sh --watch 30
#
set -eu

MONSTRO_LOG="${MONSTRO_LOG:-/var/log/monstro-netdiag.log}"
WAN_TARGET="${WAN_TARGET:-1.1.1.1}"
FE_TARGET="${FE_TARGET:-10.7.0.10}"
BE_TARGET="${BE_TARGET:-10.7.16.128}"
PING_COUNT="${PING_COUNT:-5}"
PING_DEADLINE_S="${PING_DEADLINE_S:-8}"

log() { printf '%s\n' "$*"; }
section() { printf '\n=== %s ===\n' "$*"; }

ping_avg_ms() {
  _host="$1"
  _out="$(ping -c "$PING_COUNT" -w "$PING_DEADLINE_S" "$_host" 2>/dev/null)" || true
  printf '%s\n' "$_out" | awk -F'[=/ ]' '/min\/avg\/max/ { print $8; exit }'
}

ping_loss_pct() {
  _host="$1"
  _out="$(ping -c "$PING_COUNT" -w "$PING_DEADLINE_S" "$_host" 2>/dev/null)" || true
  printf '%s\n' "$_out" | awk '/packet loss/ {
    for (i = 1; i <= NF; i++) if ($i ~ /%$/) { gsub(/%/, "", $i); print $i; exit }
  }'
}

load_1m() {
  awk '{print $1}' /proc/loadavg 2>/dev/null || echo "?"
}

cpu_busy_pct() {
  awk '/^cpu / {
    idle=$5+$6; total=0; for(i=2;i<=NF;i++) total+=$i
    if (total>0) printf "%.0f", 100*(total-idle)/total
  }' /proc/stat 2>/dev/null || echo "?"
}

conntrack_line() {
  _cur="$(cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null)" || _cur=""
  _max="$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null)" || _max=""
  if [ -n "$_cur" ] && [ -n "$_max" ] && [ "$_max" -gt 0 ] 2>/dev/null; then
    _pct=$((_cur * 100 / _max))
    printf 'conntrack=%s/%s (%s%%)' "$_cur" "$_max" "$_pct"
  else
    printf 'conntrack=n/a'
  fi
}

top_cpu_process() {
  ps -eo pcpu,pmem,comm --sort=-pcpu 2>/dev/null | awk 'NR==2 {printf "%s cpu=%s%% mem=%s%%", $3, $1, $2; exit}'
  [ $? -eq 0 ] || ps aux 2>/dev/null | awk 'NR>1 {print $3,$4,$11; exit}' | awk '{printf "%s cpu=%s%% mem=%s%%", $3, $1, $2}'
}

iface_drops_line() {
  # Sum RX/TX drop+error on non-loopback interfaces.
  ip -s link 2>/dev/null | awk '
    /^[0-9]+:/ {
      if (iface != "" && iface !~ /^lo/) {
        printf "%s:rx_drop=%d tx_drop=%d rx_err=%d tx_err=%d ", iface, rx_drop, tx_drop, rx_err, tx_err
      }
      iface=$2; gsub(":", "", iface); rx_drop=tx_drop=rx_err=tx_err=0
    }
    /RX:/{ getline; rx_drop=$4; rx_err=$3 }
    /TX:/{ getline; tx_drop=$4; tx_err=$3 }
    END {
      if (iface != "" && iface !~ /^lo/)
        printf "%s:rx_drop=%d tx_drop=%d rx_err=%d tx_err=%d", iface, rx_drop, tx_drop, rx_err, tx_err
    }
  ' | tr ' ' '\n' | grep -v '=0$' | tr '\n' ' ' | sed 's/ $//'
}

snapshot_once() {
  _ts="$(date -Iseconds 2>/dev/null || date '+%Y-%m-%dT%H:%M:%S%z')"
  _wan_avg="$(ping_avg_ms "$WAN_TARGET")"; [ -n "$_wan_avg" ] || _wan_avg="-1"
  _fe_avg="$(ping_avg_ms "$FE_TARGET")"; [ -n "$_fe_avg" ] || _fe_avg="-1"
  _be_avg="$(ping_avg_ms "$BE_TARGET")"; [ -n "$_be_avg" ] || _be_avg="-1"
  _wan_loss="$(ping_loss_pct "$WAN_TARGET")"; [ -n "$_wan_loss" ] || _wan_loss="-1"
  _fe_loss="$(ping_loss_pct "$FE_TARGET")"; [ -n "$_fe_loss" ] || _fe_loss="-1"
  _load="$(load_1m)"
  _cpu="$(cpu_busy_pct)"
  _ct="$(conntrack_line)"
  _top="$(top_cpu_process)"
  _drops="$(iface_drops_line)"
  _line="ts=$_ts load1=$_load cpu_busy=${_cpu}% $_ct top=\"$_top\" wan_avg_ms=$_wan_avg wan_loss=${_wan_loss}% fe_avg_ms=$_fe_avg fe_loss=${_fe_loss}% be_avg_ms=$_be_avg"
  [ -n "$_drops" ] && _line="$_line drops=\"$_drops\""
  _dir="$(dirname "$MONSTRO_LOG")"
  [ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true
  printf '%s\n' "$_line" >> "$MONSTRO_LOG"
  printf '%s\n' "$_line"
}

print_ping_block() {
  _label="$1"
  _host="$2"
  section "ping $_label ($_host)"
  ping -c "$PING_COUNT" -w "$PING_DEADLINE_S" "$_host" 2>/dev/null || warn "ping failed"
}

print_host_health() {
  section "host $(hostname -s 2>/dev/null || echo router)"
  log "uptime: $(uptime 2>/dev/null || true)"
  log "load: $(cat /proc/loadavg 2>/dev/null || echo n/a)"
  log "mem: $(free -h 2>/dev/null | awk '/^Mem:/ {print $3 "/" $2}' || true)"
  log "$(conntrack_line)"
  log "top cpu: $(top_cpu_process)"
  _drops="$(iface_drops_line)"
  [ -n "$_drops" ] && log "iface drops/errors: $_drops"
}

print_iptables_summary() {
  section "iptables (packet counts — hot rules first)"
  if ! command -v iptables >/dev/null 2>&1; then
    log "iptables not installed"
    return 0
  fi
  for _tbl in filter nat mangle; do
    log "--- table $_tbl FORWARD (top 12 by packets) ---"
    iptables -t "$_tbl" -L FORWARD -n -v --line-numbers 2>/dev/null \
      | awk 'NR==1 || ($1+0)>0' | head -n 13 || true
    log "--- table $_tbl PREROUTING (top 8) ---"
    iptables -t "$_tbl" -L PREROUTING -n -v --line-numbers 2>/dev/null \
      | awk 'NR==1 || ($1+0)>0' | head -n 9 || true
  done
  log "--- NAT POSTROUTING (MASQUERADE/SNAT) ---"
  iptables -t nat -L POSTROUTING -n -v --line-numbers 2>/dev/null | head -n 15 || true
}

print_processes() {
  section "top processes (cpu)"
  ps -eo pcpu,pmem,pid,user,comm,args --sort=-pcpu 2>/dev/null | head -n 12 \
    || ps aux --sort=-%cpu 2>/dev/null | head -n 12
}

print_routing() {
  section "routes to FE/BE"
  ip route get "$FE_TARGET" 2>/dev/null || true
  ip route get "$BE_TARGET" 2>/dev/null || true
  section "arp (FE)"
  ip neigh show "$FE_TARGET" 2>/dev/null || arp -n "$FE_TARGET" 2>/dev/null || true
}

print_softnet() {
  section "softnet drops (per-cpu, col3=processed col2=dropped)"
  if [ -r /proc/net/softnet_stat ]; then
    awk '{ if ($2+0 > 0) printf "cpu%02d dropped=%s processed=%s\n", NR-1, $2, $1 }' /proc/net/softnet_stat
  fi
}

print_wg() {
  command -v wg >/dev/null 2>&1 || return 0
  section "wireguard"
  wg show 2>/dev/null || true
}

interpret() {
  section "interpretation"
  _fe="$(ping_avg_ms "$FE_TARGET")"
  _wan="$(ping_avg_ms "$WAN_TARGET")"
  _fe_i="${_fe%%.*}"; _wan_i="${_wan%%.*}"
  case "$_fe" in
    ''|?*) log "fe ping failed — routing/ARP/firewall" ;;
    *)
      if [ "${_fe_i:-999}" -gt 50 ] 2>/dev/null && [ "${_wan_i:-0}" -lt 30 ] 2>/dev/null; then
        log "LAN bottleneck: WAN ok (~${_wan}ms) but FE slow (~${_fe}ms)"
        log "  check: conntrack %%, iptables FORWARD/NAT, router CPU, FE VM load"
        log "  intermittent: conntrack fill, NAT spike, ARP refresh, hypervisor scheduling"
      elif [ "${_fe_i:-0}" -gt 50 ] 2>/dev/null && [ "${_wan_i:-0}" -gt 50 ] 2>/dev/null; then
        log "WAN + LAN both slow — router overload and/or upstream"
      else
        log "ping ok from this host (wan ~${_wan}ms fe ~${_fe}ms)"
        log "  apps still slow? DNAT/nginx on FE or BE services (above L3 ping)"
      fi
      ;;
  esac
}

full_report() {
  log "monstro-netdiag full @ $(date -Iseconds 2>/dev/null || date)"
  print_host_health
  print_ping_block "WAN" "$WAN_TARGET"
  print_ping_block "FE" "$FE_TARGET"
  print_ping_block "BE" "$BE_TARGET"
  print_routing
  print_softnet
  print_iptables_summary
  print_processes
  print_wg
  interpret
}

watch_loop() {
  _sec="${1:-30}"
  log "watch every ${_sec}s (Ctrl-C to stop) — log: $MONSTRO_LOG"
  while true; do
    snapshot_once
    sleep "$_sec"
  done
}

usage() {
  sed -n '2,18p' "$0"
  printf '\nCommands:\n'
  printf '  --snapshot     one line -> %s (cron)\n' "$MONSTRO_LOG"
  printf '  --full         verbose report when sluggish\n'
  printf '  --watch [SEC]  repeat snapshot (default 30s)\n'
  printf '  --tail [N]     last N log lines\n'
}

case "${1:-}" in
  -h|--help|help) usage ;;
  --snapshot|snapshot) snapshot_once ;;
  --full|full) full_report ;;
  --watch|watch) watch_loop "${2:-30}" ;;
  --tail|tail)
    _n="${2:-20}"
    [ -r "$MONSTRO_LOG" ] && tail -n "$_n" "$MONSTRO_LOG" || log "no log yet"
    ;;
  "") full_report ;;
  *) log "unknown: $1 (try --help)"; exit 1 ;;
esac