android_cast/examples/network/wan-snapshot.sh

#!/bin/sh
#
# Log WAN vs LAN ping quality from any vantage (router, FE, Pi, …).
# Use to compare "rocketship" evenings vs slow periods (ISP/upstream vs local).
#
# Topology (see docs/INFRA.md):
#   world -> 134.17.26.161 (router 10.7.6.228) -> 10.7.0.10 (FE) -> 10.7.0.0/8 (BE, …)
#
# Install:
#   sudo cp wan-snapshot.sh /usr/local/bin/wan-snapshot.sh
#   sudo chmod +x /usr/local/bin/wan-snapshot.sh
#   sudo mkdir -p /var/log
#
# Cron (every 15 minutes — ONE entry only; script uses flock to skip overlaps):
#   */15 * * * * root /usr/local/bin/wan-snapshot.sh >>/var/log/wan-quality.cron.log 2>&1
#
# Env overrides:
#   WAN_SNAPSHOT_LOG=/var/log/wan-quality.log
#   WAN_SNAPSHOT_ROLE=auto|router|fe|pi     (default auto from hostname)
#   WAN_SNAPSHOT_EXTRAS=1                   (0 = ping only, compact v1-style line)
#   WAN_SNAPSHOT_LOCK=/var/run/wan-snapshot.lock
#   WAN_PING_TARGET=1.1.1.1
#   FE_PING_TARGET=10.7.0.10
#   BE_PING_TARGET=10.7.16.128              (set to "none" to skip)
#   ROUTER_PING_TARGET=none|10.7.6.228      (optional; useful on Pi)
#   GW_PING_TARGET=auto|none|<ip>           (default auto = default gateway)
#   PING_COUNT=10
#   PING_DEADLINE_S=15
#
set -eu

WAN_SNAPSHOT_LOG="${WAN_SNAPSHOT_LOG:-/var/log/wan-quality.log}"
WAN_SNAPSHOT_ROLE="${WAN_SNAPSHOT_ROLE:-auto}"
WAN_SNAPSHOT_EXTRAS="${WAN_SNAPSHOT_EXTRAS:-1}"
WAN_SNAPSHOT_LOCK="${WAN_SNAPSHOT_LOCK:-/var/run/wan-snapshot.lock}"
WAN_PING_TARGET="${WAN_PING_TARGET:-1.1.1.1}"
FE_PING_TARGET="${FE_PING_TARGET:-10.7.0.10}"
BE_PING_TARGET="${BE_PING_TARGET-10.7.16.128}"
ROUTER_PING_TARGET="${ROUTER_PING_TARGET:-none}"
GW_PING_TARGET="${GW_PING_TARGET:-auto}"
PING_COUNT="${PING_COUNT:-10}"
PING_DEADLINE_S="${PING_DEADLINE_S:-15}"

usage() {
  sed -n '2,28p' "$0"
  printf '\nCommands:\n'
  printf '  (default)           append one snapshot line to the log\n'
  printf '  --report            summarize today vs yesterday\n'
  printf '  --tail [N]          print last N log lines (default 24)\n'
  printf '  --correlate FILE…   merge logs by 15-min slot (role= or path basename)\n'
  printf '  --help              this text\n'
}

need_ping() {
  command -v ping >/dev/null 2>&1 || {
    echo "wan-snapshot: missing ping" >&2
    exit 1
  }
}

detect_role() {
  _role="$WAN_SNAPSHOT_ROLE"
  if [ "$_role" != "auto" ]; then
    printf '%s' "$_role"
    return 0
  fi
  _hn="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)"
  case "$_hn" in
    *raspberry*|*rpi*) printf 'pi' ;;
    *monstro*|*router*|*gw*) printf 'router' ;;
    *gentoo*|*wg0*|*fe*) printf 'fe' ;;
    *) printf 'host' ;;
  esac
}

# ping_summary HOST -> loss_pct min_ms avg_ms max_ms mdev_ms (-1 for missing)
ping_summary() {
  _host="$1"
  _out=""
  _out="$(ping -c "$PING_COUNT" -w "$PING_DEADLINE_S" "$_host" 2>/dev/null)" || true
  if [ -z "$_out" ]; then
    printf '%s %s %s %s %s' "-1" "-1" "-1" "-1" "-1"
    return 0
  fi
  _loss="$(printf '%s\n' "$_out" | awk '/packet loss/ {
    for (i = 1; i <= NF; i++) if ($i ~ /%$/) { gsub(/%/, "", $i); print $i; exit }
  }')"
  _min="-1"
  _avg="-1"
  _max="-1"
  _mdev="-1"
  _rtt="$(printf '%s\n' "$_out" | sed -n 's/.*= \([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\) ms.*/\1 \2 \3 \4/p' | head -1)"
  if [ -n "$_rtt" ]; then
    _min="${_rtt%% *}"
    _rest="${_rtt#* }"
    _avg="${_rest%% *}"
    _rest="${_rest#* }"
    _max="${_rest%% *}"
    _mdev="${_rest#* }"
  else
    _avg="$(printf '%s\n' "$_out" | awk -F'[=/ ]' '/min\/avg\/max/ { print $8; exit }')"
    [ -n "$_avg" ] || _avg="-1"
  fi
  [ -n "$_loss" ] || _loss="-1"
  printf '%s %s %s %s %s' "$_loss" "$_min" "$_avg" "$_max" "$_mdev"
}

append_ping_fields() {
  _line="$1"
  _prefix="$2"
  _host="$3"
  _sum="$(ping_summary "$_host")"
  _loss="${_sum%% *}"
  _rest="${_sum#* }"
  _min="${_rest%% *}"
  _rest="${_rest#* }"
  _avg="${_rest%% *}"
  _rest="${_rest#* }"
  _max="${_rest%% *}"
  _mdev="${_rest#* }"
  printf '%s %s=%s %s_loss=%s%% %s_min_ms=%s %s_avg_ms=%s %s_max_ms=%s %s_mdev_ms=%s' \
    "$_line" "$_prefix" "$_host" "$_prefix" "$_loss" "$_prefix" "$_min" \
    "$_prefix" "$_avg" "$_prefix" "$_max" "$_prefix" "$_mdev"
}

route_dev() {
  _host="$1"
  ip route get "$_host" 2>/dev/null | awk '{for (i = 1; i <= NF; i++) if ($i == "dev") { print $(i + 1); exit }}'
}

route_gw() {
  _host="$1"
  ip route get "$_host" 2>/dev/null | awk '{for (i = 1; i <= NF; i++) if ($i == "via") { print $(i + 1); exit }}'
}

default_gw() {
  ip route show default 2>/dev/null | awk '{print $3; exit}'
}

load_1m() {
  awk '{print $1}' /proc/loadavg 2>/dev/null || echo "n/a"
}

cpu_busy_pct() {
  awk '/^cpu / {
    idle = $5 + $6; total = 0
    for (i = 2; i <= NF; i++) total += $i
    if (total > 0) printf "%.0f", 100 * (total - idle) / total
  }' /proc/stat 2>/dev/null || echo "n/a"
}

conntrack_pct() {
  _cur="$(cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null)" || _cur=""
  _max="$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null)" || _max=""
  if [ -n "$_cur" ] && [ -n "$_max" ] && [ "$_max" -gt 0 ] 2>/dev/null; then
    _pct=$((_cur * 100 / _max))
    printf 'conntrack=%s/%s(%s%%)' "$_cur" "$_max" "$_pct"
  fi
}

neigh_state() {
  _host="$1"
  ip neigh show "$_host" 2>/dev/null | awk '{print $6; exit}'
}

iface_drops_compact() {
  ip -s link 2>/dev/null | awk '
    /^[0-9]+:/ {
      if (iface != "" && iface !~ /^lo$/) {
        if (rx_drop + tx_drop + rx_err + tx_err > 0)
          printf "%s:rd=%d,td=%d,re=%d,te=%d ", iface, rx_drop, tx_drop, rx_err, tx_err
      }
      iface = $2; gsub(":", "", iface)
      rx_drop = tx_drop = rx_err = tx_err = 0
    }
    /RX:/ { getline; rx_err = $3; rx_drop = $4 }
    /TX:/ { getline; tx_err = $3; tx_drop = $4 }
    END {
      if (iface != "" && iface !~ /^lo$/ && rx_drop + tx_drop + rx_err + tx_err > 0)
        printf "%s:rd=%d,td=%d,re=%d,te=%d", iface, rx_drop, tx_drop, rx_err, tx_err
    }
  ' | sed 's/ $//'
}

acquire_lock() {
  command -v flock >/dev/null 2>&1 || return 0
  _lock="$WAN_SNAPSHOT_LOCK"
  _dir="$(dirname "$_lock")"
  if ! mkdir -p "$_dir" 2>/dev/null || ! : >"$_lock" 2>/dev/null; then
    _lock="${TMPDIR:-/tmp}/wan-snapshot.$(id -u 2>/dev/null || echo 0).lock"
  fi
  exec 9>"$_lock" 2>/dev/null || return 0
  flock -n 9 || {
    echo "wan-snapshot: skip (already running, lock=$_lock)" >&2
    exit 0
  }
}

snapshot_once() {
  need_ping
  acquire_lock

  _ts="$(date -Iseconds 2>/dev/null || date '+%Y-%m-%dT%H:%M:%S%z')"
  _role="$(detect_role)"
  _host="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)"

  _line="ts=$_ts ver=2 role=$_role host=$_host"
  _line="$(append_ping_fields "$_line" "wan" "$WAN_PING_TARGET")"
  _line="$(append_ping_fields "$_line" "fe" "$FE_PING_TARGET")"

  if [ -n "$BE_PING_TARGET" ] && [ "$BE_PING_TARGET" != "none" ]; then
    _line="$(append_ping_fields "$_line" "be" "$BE_PING_TARGET")"
  fi

  if [ -n "$ROUTER_PING_TARGET" ] && [ "$ROUTER_PING_TARGET" != "none" ]; then
    _line="$(append_ping_fields "$_line" "router" "$ROUTER_PING_TARGET")"
  fi

  _gw_target="$GW_PING_TARGET"
  if [ "$_gw_target" = "auto" ]; then
    _gw_target="$(default_gw)"
    [ -n "$_gw_target" ] || _gw_target="none"
  fi
  if [ -n "$_gw_target" ] && [ "$_gw_target" != "none" ]; then
    _line="$(append_ping_fields "$_line" "gw" "$_gw_target")"
  fi

  if [ "$WAN_SNAPSHOT_EXTRAS" = "1" ]; then
    _line="$_line load1=$(load_1m) cpu_busy=$(cpu_busy_pct)%"
    _ct="$(conntrack_pct)"
    [ -n "$_ct" ] && _line="$_line $_ct"
    _wdev="$(route_dev "$WAN_PING_TARGET")"
    _fdev="$(route_dev "$FE_PING_TARGET")"
    [ -n "$_wdev" ] && _line="$_line route_wan=$_wdev"
    [ -n "$_fdev" ] && _line="$_line route_fe=$_fdev"
    _wgw="$(route_gw "$WAN_PING_TARGET")"
    [ -n "$_wgw" ] && _line="$_line via_wan=$_wgw"
    _nfe="$(neigh_state "$FE_PING_TARGET")"
    [ -n "$_nfe" ] && _line="$_line neigh_fe=$_nfe"
    _drops="$(iface_drops_compact)"
    [ -n "$_drops" ] && _line="$_line drops=\"$_drops\""
  fi

  _dir="$(dirname "$WAN_SNAPSHOT_LOG")"
  [ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true
  printf '%s\n' "$_line" >> "$WAN_SNAPSHOT_LOG"
  printf '%s\n' "$_line"
}

# Extract numeric field from log line (supports wan_avg_ms and legacy layouts).
field_value() {
  _line="$1"
  _field="$2"
  printf '%s\n' "$_line" | awk -v f="$_field" '
    {
      for (i = 1; i <= NF; i++) {
        if ($i ~ "^" f "=") {
          split($i, a, "=")
          v = a[2]
          gsub(/%/, "", v)
          if (v + 0 == v) { print v; exit }
        }
      }
    }
  '
}

day_stats() {
  _log="$1"
  _date="$2"
  _field="$3"
  [ -r "$_log" ] || return 0
  awk -v d="$_date" -v f="$_field" '
    $0 ~ "^ts=" d {
      for (i = 1; i <= NF; i++) {
        if ($i ~ "^" f "=") {
          split($i, a, "=")
          v = a[2]
          gsub(/%/, "", v)
          if (v + 0 == v && v >= 0) {
            n++; s += v; if (v > max) max = v
            if (v >= 50) bad++
            vals[n] = v
          }
        }
      }
    }
    END {
      if (n == 0) { print "n/a n/a n/a n/a"; exit }
      asort(vals)
      p50 = vals[int((n + 1) * 0.5)]
      p90 = vals[int(n * 0.9)]
      if (p90 == "") p90 = vals[n]
      printf "%.2f %.2f %.2f %.0f %d/%d", s / n, p50, p90, max, bad + 0, n
    }
  ' "$_log"
}

day_max_loss() {
  _log="$1"
  _date="$2"
  _field="$3"
  [ -r "$_log" ] || return 0
  awk -v d="$_date" -v f="$_field" '
    $0 ~ "^ts=" d {
      for (i = 1; i <= NF; i++) {
        if ($i ~ "^" f "=") {
          split($i, a, "=")
          v = a[2]
          gsub(/%/, "", v)
          if (v + 0 == v && v >= 0 && v > max) max = v
        }
      }
    }
    END {
      if (max == "") print "n/a"
      else printf "%.0f", max
    }
  ' "$_log"
}

report_day_block() {
  _log="$1"
  _label="$2"
  _date="$3"
  _samples="$(grep -c "^ts=$_date" "$_log" 2>/dev/null || echo 0)"
  printf '%s=%s samples=%s\n' "$_label" "$_date" "$_samples"
  for _pfx in wan fe be router gw; do
    _stats="$(day_stats "$_log" "$_date" "${_pfx}_avg_ms")"
    _mean="${_stats%% *}"
    _rest="${_stats#* }"
    _p50="${_rest%% *}"
    _rest="${_rest#* }"
    _p90="${_rest%% *}"
    _rest="${_rest#* }"
    _max="${_rest%% *}"
    _bad="${_rest#* }"
    _loss="$(day_max_loss "$_log" "$_date" "${_pfx}_loss")"
    if [ "$_mean" != "n/a" ]; then
      printf '  %s: mean=%s p50=%s p90=%s max=%s bad(>=50ms)=%s loss_max=%s%%\n' \
        "$_pfx" "$_mean" "$_p50" "$_p90" "$_max" "$_bad" "$_loss"
    fi
  done
}

report() {
  _log="$WAN_SNAPSHOT_LOG"
  _today="$(date '+%Y-%m-%d')"
  _yesterday="$(date -d 'yesterday' '+%Y-%m-%d' 2>/dev/null || date -v-1d '+%Y-%m-%d' 2>/dev/null || echo '')"

  if [ ! -r "$_log" ]; then
    echo "wan-snapshot: no log at $_log (run default command first)" >&2
    exit 1
  fi

  _role="$(detect_role)"
  printf 'log=%s role=%s\n' "$_log" "$_role"
  report_day_block "$_log" "today" "$_today"

  if [ -n "$_yesterday" ] && grep -q "^ts=$_yesterday" "$_log" 2>/dev/null; then
    report_day_block "$_log" "yesterday" "$_yesterday"
  fi

  printf '\nInterpretation:\n'
  printf '  wan bad + fe good       -> ISP/upstream or route_dev issue\n'
  printf '  wan + fe bad            -> router/LAN/tunnel path\n'
  printf '  fe bad (pi) + fe ok (fe) -> path to FE, not FE host CPU\n'
  printf '  high mdev / max vs avg  -> jitter (bufferbloat, conntrack, VPN)\n'
  printf '  conntrack>80%%           -> edge table pressure (router)\n'
  printf '  neigh_fe not REACHABLE  -> ARP/L2 flap toward FE\n'
  printf '  ping good, HTTPS slow   -> DNAT/nginx above this probe\n'
}

tail_log() {
  _n="${1:-24}"
  [ -r "$WAN_SNAPSHOT_LOG" ] || {
    echo "wan-snapshot: no log at $WAN_SNAPSHOT_LOG" >&2
    exit 1
  }
  tail -n "$_n" "$WAN_SNAPSHOT_LOG"
}

# Guess role label for correlate when log line has no role=.
log_role_label() {
  _path="$1"
  _line="$2"
  _role="$(field_value "$_line" "role")"
  if [ -n "$_role" ] && [ "$_role" != "0" ]; then
    printf '%s' "$_role"
    return 0
  fi
  case "$_path" in
    *raspberry*|*rpi*) printf 'pi' ;;
    *FE_gentoo*|*gentoo*) printf 'fe' ;;
    *monstro*|*router*) printf 'router' ;;
    *) printf '%s' "$(basename "$_path" .log)" ;;
  esac
}

# Bucket timestamp to 15-min wall-clock slot: YYYY-MM-DDTHH:00|15|30|45
slot_key() {
  _ts="$1"
  _date="${_ts%%T*}"
  _rest="${_ts#*T}"
  _hour="${_rest%%:*}"
  _min="${_rest#*:}"
  _min="${_min%%:*}"
  _min="${_min%%+*}"
  _min="${_min%%-*}"
  _bucket=$(( (_min / 15) * 15 ))
  printf '%sT%s:%02d\n' "$_date" "$_hour" "$_bucket"
}

correlate_logs() {
  if [ "$#" -lt 1 ]; then
    echo "wan-snapshot: --correlate needs at least one log file" >&2
    exit 1
  fi

  _tmp="$(mktemp "${TMPDIR:-/tmp}/wan-correlate.XXXXXX")"
  # shellcheck disable=SC2064
  trap 'rm -f "$_tmp"' EXIT INT HUP

  for _path in "$@"; do
    [ -r "$_path" ] || {
      echo "wan-snapshot: unreadable: $_path" >&2
      continue
    }
    _label="$(basename "$_path")"
    while IFS= read -r _line || [ -n "$_line" ]; do
      case "$_line" in
        ts=*) ;;
        *) continue ;;
      esac
      _ts="${_line#ts=}"
      _ts="${_ts%% *}"
      _slot="$(slot_key "$_ts")"
      _role="$(log_role_label "$_path" "$_line")"
      printf '%s\t%s\t%s\t%s\t%s\n' "$_slot" "$_role" "$_ts" "$_label" "$_line" >> "$_tmp"
    done < "$_path"
  done

  printf 'slot\t'
  _roles="$(awk -F'\t' '{print $2}' "$_tmp" | sort -u | tr '\n' ' ')"
  for _role in $_roles; do
    printf '%s_wan\t%s_fe\t%s_be\t' "$_role" "$_role" "$_role"
  done
  printf '\n'

  awk -F'\t' '
    {
      slot = $1; role = $2; ts = $3; line = $5
      key = slot SUBSEP role
      if (!(key in best_ts) || ts > best_ts[key]) {
        best_ts[key] = ts
        wan = fe = be = "-"
        for (i = 1; i <= split(line, a, " "); i++) {
          if (a[i] ~ /^wan_avg_ms=/) { split(a[i], b, "="); wan = b[2] }
          if (a[i] ~ /^fe_avg_ms=/) { split(a[i], b, "="); fe = b[2] }
          if (a[i] ~ /^be_avg_ms=/) { split(a[i], b, "="); be = b[2] }
        }
        W[slot, role, "wan"] = wan
        W[slot, role, "fe"] = fe
        W[slot, role, "be"] = be
        slots[slot] = 1
        roles[role] = 1
      }
    }
    END {
      nroles = 0
      for (r in roles) rolelist[++nroles] = r
      asort(rolelist)
      nslots = 0
      for (s in slots) slotlist[++nslots] = s
      asort(slotlist)
      for (si = 1; si <= nslots; si++) {
        s = slotlist[si]
        printf "%s", s
        for (ri = 1; ri <= nroles; ri++) {
          r = rolelist[ri]
          printf "\t%s\t%s\t%s", W[s, r, "wan"] + 0 ? W[s, r, "wan"] : "-", \
            W[s, r, "fe"] + 0 ? W[s, r, "fe"] : "-", \
            W[s, r, "be"] + 0 ? W[s, r, "be"] : "-"
        }
        printf "\n"
      }
    }
  ' "$_tmp" | while IFS= read -r _row; do
    printf '%s\n' "$_row"
  done
}

case "${1:-}" in
  -h|--help|help)
    usage
    ;;
  --report|report)
    report
    ;;
  --tail|tail)
    tail_log "${2:-24}"
    ;;
  --correlate|correlate)
    shift
    correlate_logs "$@"
    ;;
  "")
    snapshot_once
    ;;
  *)
    echo "wan-snapshot: unknown command: $1 (try --help)" >&2
    exit 1
    ;;
esac