mirror of
git://f0xx.org/android_cast
synced 2026-07-29 04:18:09 +03:00
sync on be, vpn
This commit is contained in:
Anton Afanasyeu
@@ -81,6 +81,14 @@
|
||||
return d.innerHTML;
|
||||
}
|
||||
|
||||
function formatBytes(n) {
|
||||
const b = Number(n) || 0;
|
||||
if (b < 1024) return b + ' B';
|
||||
if (b < 1024 * 1024) return (b / 1024).toFixed(1) + ' KiB';
|
||||
if (b < 1024 * 1024 * 1024) return (b / (1024 * 1024)).toFixed(2) + ' MiB';
|
||||
return (b / (1024 * 1024 * 1024)).toFixed(2) + ' GiB';
|
||||
}
|
||||
|
||||
function getColWidths() {
|
||||
try {
|
||||
const w = JSON.parse(lsGet(STORAGE.colWidths, '{}'));
|
||||
@@ -393,10 +401,26 @@
|
||||
if (d.sdk_int != null) lines.push('Android API: ' + d.sdk_int);
|
||||
if (d.os_release) lines.push('Android version: ' + d.os_release);
|
||||
if (d.abis && d.abis.length) lines.push('ABIs: ' + formatAbis(d.abis));
|
||||
if (d.wan_ip) lines.push('WAN IP (last poll): ' + d.wan_ip);
|
||||
if (d.lan_ip) lines.push('LAN IP: ' + d.lan_ip);
|
||||
if (d.lan_ip) lines.push('Device LAN IP: ' + d.lan_ip);
|
||||
if (d.device_wan_ip) lines.push('Device WAN IP: ' + d.device_wan_ip);
|
||||
if (d.poll_source_ip) {
|
||||
let pollLabel = 'Poll source IP (server view)';
|
||||
if (d.poll_source_is_private) {
|
||||
pollLabel += ' — intra/private hop, not device WAN';
|
||||
}
|
||||
lines.push(pollLabel + ': ' + d.poll_source_ip);
|
||||
}
|
||||
if (d.vpn_ip) lines.push('VPN IP: ' + d.vpn_ip);
|
||||
if (d.vpn_public_key) lines.push('VPN public key: ' + d.vpn_public_key);
|
||||
if (d.wg_rx_bytes != null || d.wg_tx_bytes != null) {
|
||||
const rx = formatBytes(d.wg_rx_bytes || 0);
|
||||
const tx = formatBytes(d.wg_tx_bytes || 0);
|
||||
lines.push('VPN traffic (BE wg): ↓' + rx + ' ↑' + tx);
|
||||
}
|
||||
if (d.wg_latest_handshake > 0) {
|
||||
lines.push('VPN last handshake: ' + new Date(d.wg_latest_handshake * 1000).toISOString());
|
||||
}
|
||||
if (d.wg_endpoint) lines.push('VPN endpoint: ' + d.wg_endpoint);
|
||||
lines.push('App version: ' + (d.app_version || '—'));
|
||||
lines.push('Opt-in mode: ' + (d.opt_in_mode || 'none'));
|
||||
lines.push('Last seen: ' + (d.last_seen_at || '—'));
|
||||
|
||||
@@ -80,8 +80,14 @@ else
|
||||
note "php CLI cannot run wg genkey — ensure www-data can run wg if provision_peers=true"
|
||||
fi
|
||||
|
||||
# DB tables (via bootstrap)
|
||||
"$PHP" -r '
|
||||
# DB tables (via bootstrap) — use FPM user + php81 so PDO/mysql extensions match production
|
||||
DB_RUN="$PHP"
|
||||
DB_USER=""
|
||||
if [[ "$(id -u)" -eq 0 ]] && id nginx >/dev/null 2>&1; then
|
||||
DB_RUN="sudo -u nginx $PHP"
|
||||
DB_USER="nginx"
|
||||
fi
|
||||
$DB_RUN -r '
|
||||
require "'"$ROOT"'/src/bootstrap.php";
|
||||
RemoteAccessRepository::ensureSchema();
|
||||
$pdo = Database::pdo();
|
||||
|
||||
@@ -266,11 +266,12 @@ final class RemoteAccessRepository {
|
||||
$pdo = Database::pdo();
|
||||
$now = self::nowSql();
|
||||
$pdo->prepare(
|
||||
'UPDATE remote_access_devices SET opt_in_mode = ?, last_random = ?, app_version = ?, last_seen_at = ?, updated_at = ? WHERE device_id = ?'
|
||||
'UPDATE remote_access_devices SET whitelisted = 0, opt_in_mode = ?, last_random = ?, app_version = ?, last_seen_at = ?, updated_at = ? WHERE device_id = ?'
|
||||
)->execute([self::OPT_NONE, $random, $appVersion, $now, $now, $deviceId]);
|
||||
|
||||
self::closeActiveSessionsForDevice($deviceId, 'user_disabled');
|
||||
self::logEvent($deviceId, null, null, 'user_disable', 'device_opt_out', null, $clientIp);
|
||||
self::logEvent($deviceId, null, null, 'whitelist_revoked', 'device_opt_out', null, $clientIp);
|
||||
|
||||
return self::raEnvelope('disabled', ['reason' => 'user_opt_out']);
|
||||
}
|
||||
@@ -667,7 +668,9 @@ final class RemoteAccessRepository {
|
||||
$sql .= ' WHERE company_id = ?';
|
||||
$params[] = $companyId;
|
||||
}
|
||||
$sql .= ' ORDER BY last_seen_at DESC, device_id ASC';
|
||||
$sql .= Database::isMysql()
|
||||
? ' ORDER BY last_seen_at DESC NULLS LAST, device_id ASC'
|
||||
: ' ORDER BY datetime(COALESCE(last_seen_at, \'1970-01-01\')) DESC, device_id ASC';
|
||||
$st = Database::pdo()->prepare($sql);
|
||||
$st->execute($params);
|
||||
return $st->fetchAll(PDO::FETCH_ASSOC) ?: [];
|
||||
@@ -845,11 +848,17 @@ final class RemoteAccessRepository {
|
||||
$row['os_release'] = $ctx['os_release'] ?? '';
|
||||
$row['sdk_int'] = $ctx['sdk_int'] ?? null;
|
||||
$row['abis'] = $ctx['abis'] ?? [];
|
||||
$row['wan_ip'] = self::resolveDeviceWanIp($row, $storedMeta);
|
||||
$row['lan_ip'] = self::resolveDeviceLanIp($storedMeta);
|
||||
$row['poll_source_ip'] = self::resolvePollSourceIp($row);
|
||||
$row['poll_source_is_private'] = self::isPrivateIp($row['poll_source_ip']);
|
||||
$row['device_wan_ip'] = trim((string) ($storedMeta['wan_ip'] ?? ''));
|
||||
$net = self::lookupDeviceSessionNetwork($deviceId);
|
||||
$row['vpn_ip'] = $net['vpn_ip'] ?? '';
|
||||
$row['vpn_public_key'] = $net['vpn_public_key'] ?? '';
|
||||
$row['wg_rx_bytes'] = $net['wg_rx_bytes'] ?? 0;
|
||||
$row['wg_tx_bytes'] = $net['wg_tx_bytes'] ?? 0;
|
||||
$row['wg_latest_handshake'] = $net['wg_latest_handshake'] ?? 0;
|
||||
$row['wg_endpoint'] = $net['wg_endpoint'] ?? '';
|
||||
$row['issue_count'] = $ctx['issue_count'] ?? 0;
|
||||
$row['graph_session_count'] = $ctx['graph_session_count'] ?? 0;
|
||||
$row['latest_issue_id'] = $ctx['latest_issue_id'] ?? null;
|
||||
@@ -911,12 +920,8 @@ final class RemoteAccessRepository {
|
||||
return (time() - $t) <= $maxAgeSec;
|
||||
}
|
||||
|
||||
/** @param array<string, mixed> $storedMeta */
|
||||
private static function resolveDeviceWanIp(array $row, array $storedMeta): string {
|
||||
$fromDevice = trim((string) ($storedMeta['wan_ip'] ?? ''));
|
||||
if ($fromDevice !== '') {
|
||||
return self::normalizeClientIp($fromDevice);
|
||||
}
|
||||
/** @param array<string, mixed> $row */
|
||||
private static function resolvePollSourceIp(array $row): string {
|
||||
$fromRow = trim((string) ($row['last_client_ip'] ?? ''));
|
||||
if ($fromRow !== '') {
|
||||
return self::normalizeClientIp($fromRow);
|
||||
@@ -936,6 +941,17 @@ final class RemoteAccessRepository {
|
||||
return self::normalizeClientIp($ip);
|
||||
}
|
||||
|
||||
private static function isPrivateIp(string $ip): bool {
|
||||
if ($ip === '' || !filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) {
|
||||
return false;
|
||||
}
|
||||
if (!filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4 | FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/** @param array<string, mixed> $storedMeta */
|
||||
private static function resolveDeviceLanIp(array $storedMeta): string {
|
||||
return trim((string) ($storedMeta['lan_ip'] ?? ''));
|
||||
@@ -953,9 +969,16 @@ final class RemoteAccessRepository {
|
||||
return $raw;
|
||||
}
|
||||
|
||||
/** @return array{vpn_ip: string, vpn_public_key: string} */
|
||||
/** @return array{vpn_ip: string, vpn_public_key: string, wg_rx_bytes: int, wg_tx_bytes: int, wg_latest_handshake: int, wg_endpoint: string} */
|
||||
private static function lookupDeviceSessionNetwork(string $deviceId): array {
|
||||
$empty = ['vpn_ip' => '', 'vpn_public_key' => ''];
|
||||
$empty = [
|
||||
'vpn_ip' => '',
|
||||
'vpn_public_key' => '',
|
||||
'wg_rx_bytes' => 0,
|
||||
'wg_tx_bytes' => 0,
|
||||
'wg_latest_handshake' => 0,
|
||||
'wg_endpoint' => '',
|
||||
];
|
||||
if ($deviceId === '') {
|
||||
return $empty;
|
||||
}
|
||||
@@ -978,9 +1001,15 @@ final class RemoteAccessRepository {
|
||||
$vpnIp = $addr !== '' ? preg_replace('/\/\d+$/', '', $addr) ?? $addr : '';
|
||||
$pub = trim((string) ($row['wg_client_public_key'] ?? ''));
|
||||
|
||||
$stats = $pub !== '' ? WireGuardTrafficStats::forPublicKey($pub) : null;
|
||||
|
||||
return [
|
||||
'vpn_ip' => is_string($vpnIp) ? trim($vpnIp) : '',
|
||||
'vpn_public_key' => $pub,
|
||||
'wg_rx_bytes' => (int) ($stats['rx_bytes'] ?? 0),
|
||||
'wg_tx_bytes' => (int) ($stats['tx_bytes'] ?? 0),
|
||||
'wg_latest_handshake' => (int) ($stats['latest_handshake'] ?? 0),
|
||||
'wg_endpoint' => (string) ($stats['endpoint'] ?? ''),
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
<?php
|
||||
declare(strict_types=1);
|
||||
|
||||
/** Live WireGuard transfer counters from `wg show dump` (BE data plane). */
|
||||
final class WireGuardTrafficStats {
|
||||
private function __construct() {}
|
||||
|
||||
/** @return array<string, array{rx_bytes: int, tx_bytes: int, latest_handshake: int, endpoint: string}> */
|
||||
public static function peerMap(): array {
|
||||
static $cache = null;
|
||||
static $cacheAt = 0;
|
||||
if ($cache !== null && (time() - $cacheAt) < 5) {
|
||||
return $cache;
|
||||
}
|
||||
$cache = self::loadPeerMap();
|
||||
$cacheAt = time();
|
||||
return $cache;
|
||||
}
|
||||
|
||||
/** @return array{rx_bytes: int, tx_bytes: int, latest_handshake: int, endpoint: string}|null */
|
||||
public static function forPublicKey(string $publicKey): ?array {
|
||||
$pub = trim($publicKey);
|
||||
if ($pub === '') {
|
||||
return null;
|
||||
}
|
||||
return self::peerMap()[$pub] ?? null;
|
||||
}
|
||||
|
||||
/** @return array<string, array{rx_bytes: int, tx_bytes: int, latest_handshake: int, endpoint: string}> */
|
||||
private static function loadPeerMap(): array {
|
||||
$iface = (string) cfg('remote_access.wg_interface', 'wg0');
|
||||
$prefix = rtrim((string) cfg('remote_access.wg_set_prefix', ''), " \t");
|
||||
$cmd = ($prefix !== '' ? $prefix . ' ' : '') . 'wg show ' . escapeshellarg($iface) . ' dump 2>/dev/null';
|
||||
$raw = @shell_exec($cmd);
|
||||
if (!is_string($raw) || trim($raw) === '') {
|
||||
return [];
|
||||
}
|
||||
$out = [];
|
||||
foreach (preg_split('/\r\n|\n|\r/', trim($raw)) as $line) {
|
||||
if ($line === '' || str_starts_with($line, 'interface:')) {
|
||||
continue;
|
||||
}
|
||||
$cols = explode("\t", $line);
|
||||
if (count($cols) < 7) {
|
||||
continue;
|
||||
}
|
||||
$pub = trim($cols[0]);
|
||||
if ($pub === '') {
|
||||
continue;
|
||||
}
|
||||
$out[$pub] = [
|
||||
'rx_bytes' => (int) ($cols[5] ?? 0),
|
||||
'tx_bytes' => (int) ($cols[6] ?? 0),
|
||||
'latest_handshake' => (int) ($cols[4] ?? 0),
|
||||
'endpoint' => trim((string) ($cols[2] ?? '')),
|
||||
];
|
||||
}
|
||||
return $out;
|
||||
}
|
||||
|
||||
public static function formatBytes(int $bytes): string {
|
||||
if ($bytes < 1024) {
|
||||
return $bytes . ' B';
|
||||
}
|
||||
if ($bytes < 1024 * 1024) {
|
||||
return round($bytes / 1024, 1) . ' KiB';
|
||||
}
|
||||
if ($bytes < 1024 * 1024 * 1024) {
|
||||
return round($bytes / (1024 * 1024), 2) . ' MiB';
|
||||
}
|
||||
return round($bytes / (1024 * 1024 * 1024), 2) . ' GiB';
|
||||
}
|
||||
}
|
||||
@@ -68,6 +68,7 @@ require_once __DIR__ . '/UrlShortenerDatabase.php';
|
||||
require_once __DIR__ . '/ShortLinksRepository.php';
|
||||
require_once __DIR__ . '/WireGuardPeerProvisioner.php';
|
||||
require_once __DIR__ . '/WireGuardAddressPool.php';
|
||||
require_once __DIR__ . '/WireGuardTrafficStats.php';
|
||||
require_once __DIR__ . '/RsshSessionProvisioner.php';
|
||||
require_once __DIR__ . '/AnalyticsHead.php';
|
||||
|
||||
|
||||
@@ -443,6 +443,7 @@
|
||||
Devices appear after they poll remote access (dev settings → WireGuard or RSSH). Sort is by <strong>last seen</strong> — top rows are actively polling.
|
||||
Whitelist rows marked <span class="tag-pill tag-pill--warn">Needs whitelist</span> (opted in, not yet allowed).
|
||||
Stale rows have not polled in 7+ days.
|
||||
<strong>Poll source IP</strong> is the address BE sees on the HTTP request (often a router or proxy on <code>10.7.x.x</code>) — not the device WAN or LAN.
|
||||
</p>
|
||||
<div class="reports-table-wrap">
|
||||
<table class="data-table reports-tree reports-table--cols" id="ra-devices-table">
|
||||
|
||||
@@ -1,204 +1,92 @@
|
||||
# Edge network probes
|
||||
# Edge network watchdog
|
||||
|
||||
Scripts for **router**, **FE**, and **external vantage** (e.g. Raspberry Pi) — not BE app hosts.
|
||||
**One script:** `wan-watch.sh` — deploy everywhere, read one log, get a verdict every 5 minutes.
|
||||
|
||||
## Auto watchdog (v2.1)
|
||||
|
||||
When a cron snapshot exceeds thresholds, `wan-snapshot.sh` appends **alert** + **deep** lines to the same log (`WAN_WATCH_LOG`).
|
||||
|
||||
| `kind=` | Meaning |
|
||||
|---------|---------|
|
||||
| `snapshot` | Normal 15-min ping row |
|
||||
| `alert` | Threshold breached; lists `triggers=` |
|
||||
| `deep` | One parseable fact (`section=host`, `burst`, `iface`, `iptables`, …) |
|
||||
|
||||
**Deploy both files:**
|
||||
## Deploy
|
||||
|
||||
```bash
|
||||
sudo cp examples/network/wan-snapshot.sh examples/network/wan-deep-diag.sh /usr/local/bin/
|
||||
sudo chmod +x /usr/local/bin/wan-snapshot.sh /usr/local/bin/wan-deep-diag.sh
|
||||
sudo cp examples/network/wan-watch.sh /usr/local/bin/wan-watch.sh
|
||||
sudo chmod +x /usr/local/bin/wan-watch.sh
|
||||
sudo cp examples/network/wan-watch.env.example /etc/conf.d/wan-watch
|
||||
# edit WAN_WATCH_ROLE=pi|router|fe
|
||||
```
|
||||
|
||||
**Env** (`/etc/conf.d/wan-snapshot`):
|
||||
Remove old cron entries for `wan-snapshot.sh` / `monstro-netdiag.sh`. **One cron only:**
|
||||
|
||||
```cron
|
||||
*/5 * * * * root . /etc/conf.d/wan-watch 2>/dev/null; /usr/local/bin/wan-watch.sh >>/var/log/wan-watch.cron.log 2>&1
|
||||
```
|
||||
|
||||
Log: `/var/log/wan-watch.log`
|
||||
|
||||
## Every run (5 min) captures
|
||||
|
||||
| Layer | What |
|
||||
|-------|------|
|
||||
| **Ping matrix** | wan, fe, be, gw, router — loss/min/avg/max/mdev |
|
||||
| **Verdict** | `kind=verdict` — immediate conclusion + action hint |
|
||||
| **Tunnels** | GRE links/modules, PPTP (1723/pppd/pptpd), **tinc** (tincctl status), **WireGuard**, **iodine/dns0/tun0** |
|
||||
| **Edge services** | named, squid, nginx, iodined — proc count + CPU |
|
||||
| **Host** | load, cpu, conntrack, softnet drops, ss summary |
|
||||
| **Network** | routes, neigh, iface drops, qdisc backlog, iptables hot rules (router) |
|
||||
| **On anomaly** | `kind=alert` + `kind=deep` burst pings + full probe repeat (5 min cooldown) |
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
wan-watch.sh # cron default
|
||||
wan-watch.sh --full # instant human dump (when slow NOW)
|
||||
wan-watch.sh --analyze # final conclusion from log (last 24h) — no waiting
|
||||
wan-watch.sh --analyze 6 # last 6 hours
|
||||
wan-watch.sh --timeline # hourly good/bad bars
|
||||
wan-watch.sh --correlate /var/log/wan-watch.log path/to/other/log
|
||||
wan-watch.sh --force-deep # full capture even if OK
|
||||
wan-watch.sh --tail 30
|
||||
```
|
||||
|
||||
## Verdicts (automatic)
|
||||
|
||||
| verdict | Meaning |
|
||||
|---------|---------|
|
||||
| `OK` | Within thresholds |
|
||||
| `MONSTRO_WAN_BLOAT` | Router WAN slow, FE local OK — **your main issue** |
|
||||
| `UPSTREAM_GW` | WAN + gw 10.7.16.228 both slow |
|
||||
| `PATH_TO_FE` | Pi ISP OK, path to 10.7.0.10 slow |
|
||||
| `CONNTRACK_PRESSURE` | nf_conntrack > 70% |
|
||||
| `TUNNEL_DNS0_LOAD` | iodine dns0 drops elevated |
|
||||
| `WAN_JITTER` | High mdev, bufferbloat signal |
|
||||
|
||||
Cron stdout example:
|
||||
|
||||
```text
|
||||
WAN-WATCH [router/monstro] verdict=MONSTRO_WAN_BLOAT confidence=high summary="WAN leg slow..." action="SQM/fq_codel on xenbr0..."
|
||||
```
|
||||
|
||||
## `--analyze` example output
|
||||
|
||||
```text
|
||||
CONCLUSION: MONSTRO WAN EDGE QUEUEING (confidence=HIGH)
|
||||
Fix order: 1) fq_codel/SQM on xenbr0 2) gw 10.7.16.228 3) squid/named 4) iodine dns0
|
||||
```
|
||||
|
||||
Run this **any time** — no need to wait 24h after deploy.
|
||||
|
||||
## Host config (`/etc/conf.d/wan-watch`)
|
||||
|
||||
```sh
|
||||
WAN_WATCH_LOG=/var/log/wan-watch.log
|
||||
WAN_SNAPSHOT_LOG=/var/log/wan-watch.log
|
||||
WAN_SNAPSHOT_WATCH=1
|
||||
WAN_WATCH_ROLE=pi # or router on monstro
|
||||
ROUTER_PING_TARGET=10.7.6.228 # Pi only
|
||||
WAN_WATCH_DEEP=auto
|
||||
WAN_WATCH_COOLDOWN_S=300
|
||||
```
|
||||
|
||||
**Triggers (defaults):** `*_avg_ms >= 50`, `*_loss >= 1%`, `mdev >= 80`, `conntrack >= 80%`, `neigh_fe` not REACHABLE/DELAY, plus role hints `pi_fe_path` / `router_wan_path`.
|
||||
## Legacy
|
||||
|
||||
**Cooldown:** one deep capture per 14 min (`WAN_WATCH_COOLDOWN_S=840`) to avoid cron pile-up.
|
||||
- `wan-snapshot.sh` → wrapper to `wan-watch.sh`
|
||||
- `wan-deep-diag.sh` → merged into `wan-watch.sh`
|
||||
- `monstro-netdiag.sh` → use `wan-watch.sh --full` instead
|
||||
|
||||
**Review after a bad spell:**
|
||||
## Known conclusion (from your logs)
|
||||
|
||||
```bash
|
||||
/usr/local/bin/wan-snapshot.sh --alerts 80
|
||||
/usr/local/bin/wan-snapshot.sh --correlate /var/log/wan-watch.log …
|
||||
grep 'kind=deep.*section=burst' /var/log/wan-watch.log | tail
|
||||
grep 'kind=deep.*section=iptables' /var/log/wan-watch.log | tail
|
||||
```
|
||||
|
||||
Example deep lines:
|
||||
|
||||
```text
|
||||
kind=alert ts=2026-06-12T21:45:01+03:00 role=router host=monstro parent_ts=... triggers="wan_avg=535.365,router_wan_path"
|
||||
kind=deep ts=2026-06-12T21:45:01+03:00 role=router host=monstro parent_ts=... section=burst label=wan target=1.1.1.1 loss=0% avg_ms=520 max_ms=790 mdev_ms=110
|
||||
kind=deep ts=2026-06-12T21:45:01+03:00 role=router host=monstro parent_ts=... section=iptables tbl=filter chain=FORWARD line=3 pkts=188000 target=ACCEPT
|
||||
```
|
||||
|
||||
## `wan-snapshot.sh` (v2)
|
||||
|
||||
Periodic ping comparison plus lightweight host/path context.
|
||||
|
||||
| Target | Default | Meaning |
|
||||
|--------|---------|---------|
|
||||
| **WAN** | `1.1.1.1` | ISP/upstream leg |
|
||||
| **FE** | `10.7.0.10` | Path to Gentoo FE |
|
||||
| **BE** | `10.7.16.128` | Deeper LAN (disable: `BE_PING_TARGET=none`) |
|
||||
| **router** | `none` | Optional modem ping (`ROUTER_PING_TARGET` on Pi) |
|
||||
| **gw** | auto | Default gateway ping |
|
||||
|
||||
Each target logs: `loss`, `min`, `avg`, `max`, `mdev` (jitter).
|
||||
|
||||
Extras (`WAN_SNAPSHOT_EXTRAS=1`): `load1`, `cpu_busy`, `conntrack`, `route_wan`/`route_fe`, `via_wan`, `neigh_fe`, non-zero `drops`.
|
||||
|
||||
Log format: `ver=2 role=pi|fe|router host=…` — older lines without `ver=` still work in `--report` / `--correlate`.
|
||||
|
||||
### Install (any host)
|
||||
|
||||
```bash
|
||||
sudo cp examples/network/wan-snapshot.sh /usr/local/bin/wan-snapshot.sh
|
||||
sudo chmod +x /usr/local/bin/wan-snapshot.sh
|
||||
sudo cp examples/network/wan-snapshot.env.example /etc/conf.d/wan-snapshot
|
||||
# edit /etc/conf.d/wan-snapshot for role-specific targets
|
||||
```
|
||||
|
||||
### Cron (every 15 minutes — **one** entry per host)
|
||||
|
||||
```cron
|
||||
*/15 * * * * root . /etc/conf.d/wan-snapshot 2>/dev/null; /usr/local/bin/wan-snapshot.sh >>/var/log/wan-quality.cron.log 2>&1
|
||||
```
|
||||
|
||||
The script uses `flock` on `/var/run/wan-snapshot.lock` so duplicate crontab lines do not double-log.
|
||||
|
||||
Log: `/var/log/wan-quality.log` (override with `WAN_SNAPSHOT_LOG`).
|
||||
|
||||
### Suggested `/etc/conf.d/wan-snapshot`
|
||||
|
||||
**Raspberry Pi** (path into infra matters most):
|
||||
|
||||
```sh
|
||||
WAN_SNAPSHOT_ROLE=pi
|
||||
ROUTER_PING_TARGET=10.7.6.228
|
||||
FE_PING_TARGET=10.7.0.10
|
||||
BE_PING_TARGET=10.7.16.128
|
||||
```
|
||||
|
||||
**Gentoo FE**:
|
||||
|
||||
```sh
|
||||
WAN_SNAPSHOT_ROLE=fe
|
||||
FE_PING_TARGET=10.7.0.10
|
||||
BE_PING_TARGET=10.7.16.128
|
||||
ROUTER_PING_TARGET=10.7.6.228
|
||||
```
|
||||
|
||||
**Debian router**:
|
||||
|
||||
```sh
|
||||
WAN_SNAPSHOT_ROLE=router
|
||||
FE_PING_TARGET=10.7.0.10
|
||||
BE_PING_TARGET=10.7.16.128
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
```bash
|
||||
# one line appended + printed
|
||||
sudo /usr/local/bin/wan-snapshot.sh
|
||||
|
||||
# today vs yesterday (mean, p50, p90, bad>=50ms counts)
|
||||
/usr/local/bin/wan-snapshot.sh --report
|
||||
|
||||
# recent samples
|
||||
/usr/local/bin/wan-snapshot.sh --tail 48
|
||||
|
||||
# merge Pi + FE (+ router) logs by 15-min slot
|
||||
/usr/local/bin/wan-snapshot.sh --correlate \
|
||||
tmp/raspberrypi/var/log/wan-quality.log \
|
||||
tmp/FE_gentoo/var/log/wan-quality.log
|
||||
```
|
||||
|
||||
Example v2 line:
|
||||
|
||||
```text
|
||||
ts=2026-06-12T15:30:01+03:00 ver=2 role=fe host=wg0 wan=1.1.1.1 wan_loss=0% wan_min_ms=9.1 wan_avg_ms=10.6 wan_max_ms=12.4 wan_mdev_ms=0.8 fe=10.7.0.10 fe_loss=0% fe_min_ms=0.05 fe_avg_ms=0.07 fe_max_ms=0.10 fe_mdev_ms=0.01 be=10.7.16.128 be_loss=0% be_avg_ms=0.20 load1=0.12 cpu_busy=4% route_wan=eno0 route_fe=lo neigh_fe=REACHABLE
|
||||
```
|
||||
|
||||
### Reading results
|
||||
|
||||
| Pattern | Likely cause |
|
||||
|---------|----------------|
|
||||
| `wan` bad, `fe` good (on router) | ISP/upstream |
|
||||
| `wan` + `fe` bad on router | modem CPU, conntrack, LAN |
|
||||
| `fe` bad on **pi**, `fe` good on **fe** | path/tunnel to FE, not FE VM |
|
||||
| `wan_max_ms` >> `wan_avg_ms` or high `mdev` | jitter / bufferbloat |
|
||||
| `conntrack>80%` | edge table full |
|
||||
| `neigh_fe` not `REACHABLE` | ARP/L2 toward FE |
|
||||
| `route_fe=tun0` / `ppp0` on Pi | metric is tunnel quality |
|
||||
| ping good, HTTPS slow | DNAT/nginx (not measured here) |
|
||||
|
||||
From **outside** LAN, still run occasionally:
|
||||
|
||||
```bash
|
||||
mtr -rwzc 50 134.17.26.161
|
||||
```
|
||||
|
||||
## `monstro-netdiag.sh` (bottleneck + iptables + processes)
|
||||
|
||||
Run on the **Debian edge router** (`10.7.6.228`) when infra feels stuck. Complements `wan-snapshot` with on-demand depth.
|
||||
|
||||
```bash
|
||||
sudo cp examples/network/monstro-netdiag.sh /usr/local/bin/
|
||||
sudo chmod +x /usr/local/bin/monstro-netdiag.sh
|
||||
```
|
||||
|
||||
**Cron** (optional; pair with wan-snapshot on router only):
|
||||
|
||||
```cron
|
||||
*/15 * * * * root /usr/local/bin/monstro-netdiag.sh --snapshot >>/var/log/monstro-netdiag.cron.log 2>&1
|
||||
```
|
||||
|
||||
**When sluggish**:
|
||||
|
||||
```bash
|
||||
sudo monstro-netdiag.sh --full
|
||||
sudo monstro-netdiag.sh --watch 30
|
||||
monstro-netdiag.sh --tail 48
|
||||
```
|
||||
|
||||
### Why intermittent (excluding ISP)?
|
||||
|
||||
| Cause | What to look for |
|
||||
|-------|------------------|
|
||||
| **conntrack table full** | `conntrack>80%` in wan-snapshot or monstro |
|
||||
| **Router CPU / softirq** | high `cpu_busy`, `softnet dropped` in `--full` |
|
||||
| **iptables rule cost** | FORWARD/NAT rules with huge packet counts |
|
||||
| **FE/BE VM host busy** | FE ping high from router, low `load1` on router |
|
||||
| **ARP/route flap** | `neigh_fe` not REACHABLE |
|
||||
| **Bufferbloat on LAN** | high `mdev` / `max`, 0% loss |
|
||||
|
||||
Good vs bad: save one `--full` output on a rocketship evening and diff when stuck.
|
||||
|
||||
### Safe checks on monstro (read-only)
|
||||
|
||||
```bash
|
||||
cat /proc/sys/net/netfilter/nf_conntrack_count
|
||||
cat /proc/sys/net/netfilter/nf_conntrack_max
|
||||
iptables -L FORWARD -n -v --line-numbers | head
|
||||
ip neigh show 10.7.0.10
|
||||
```
|
||||
|
||||
Do **not** change iptables/NAT on monstro without a rollback plan (per infra policy).
|
||||
**MONSTRO WAN EDGE QUEUEING** on `xenbr0` → upstream `10.7.16.228`. Not FE/BE, not Pi ISP. Contributing load: **named**, **squid**, **dns0** (iodine). Fix: **SQM/fq_codel** on WAN iface first.
|
||||
|
||||
@@ -1,696 +1,4 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Log WAN vs LAN ping quality from any vantage (router, FE, Pi, …).
|
||||
# Use to compare "rocketship" evenings vs slow periods (ISP/upstream vs local).
|
||||
#
|
||||
# Topology (see docs/INFRA.md):
|
||||
# world -> 134.17.26.161 (router 10.7.6.228) -> 10.7.0.10 (FE) -> 10.7.0.0/8 (BE, …)
|
||||
#
|
||||
# Install:
|
||||
# sudo cp wan-snapshot.sh /usr/local/bin/wan-snapshot.sh
|
||||
# sudo chmod +x /usr/local/bin/wan-snapshot.sh
|
||||
# sudo mkdir -p /var/log
|
||||
#
|
||||
# Cron (every 15 minutes — ONE entry only; script uses flock to skip overlaps):
|
||||
# */15 * * * * root /usr/local/bin/wan-snapshot.sh >>/var/log/wan-quality.cron.log 2>&1
|
||||
#
|
||||
# Env overrides:
|
||||
# WAN_WATCH_LOG=/var/log/wan-watch.log (unified: snapshot + alert + deep)
|
||||
# WAN_SNAPSHOT_LOG=/var/log/wan-quality.log (legacy snapshot copy if different)
|
||||
# WAN_SNAPSHOT_ROLE=auto|router|fe|pi
|
||||
# WAN_SNAPSHOT_EXTRAS=1
|
||||
# WAN_SNAPSHOT_WATCH=1 (0 = disable auto deep-diag on anomaly)
|
||||
# WAN_SNAPSHOT_LOCK=/var/run/wan-snapshot.lock
|
||||
# WAN_WATCH_COOLDOWN_S=840 (min seconds between deep captures)
|
||||
# WAN_ALERT_MS=50 FE_ALERT_MS=50 BE_ALERT_MS=50 GW_ALERT_MS=50
|
||||
# WAN_LOSS_ALERT_PCT=1 WAN_MDEV_ALERT_MS=80 WAN_CONNTRACK_ALERT_PCT=80
|
||||
# WAN_PING_TARGET=1.1.1.1 FE_PING_TARGET=10.7.0.10
|
||||
# BE_PING_TARGET=10.7.16.128 ROUTER_PING_TARGET=none GW_PING_TARGET=auto
|
||||
# PING_COUNT=10 PING_DEADLINE_S=15
|
||||
#
|
||||
# Install deep-diag library next to this script:
|
||||
# cp wan-snapshot.sh wan-deep-diag.sh /usr/local/bin/
|
||||
#
|
||||
set -eu
|
||||
|
||||
WAN_SNAPSHOT_LOG="${WAN_SNAPSHOT_LOG:-/var/log/wan-quality.log}"
|
||||
WAN_WATCH_LOG="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
|
||||
WAN_SNAPSHOT_ROLE="${WAN_SNAPSHOT_ROLE:-auto}"
|
||||
WAN_SNAPSHOT_EXTRAS="${WAN_SNAPSHOT_EXTRAS:-1}"
|
||||
WAN_SNAPSHOT_WATCH="${WAN_SNAPSHOT_WATCH:-1}"
|
||||
WAN_SNAPSHOT_LOCK="${WAN_SNAPSHOT_LOCK:-/var/run/wan-snapshot.lock}"
|
||||
WAN_WATCH_COOLDOWN_S="${WAN_WATCH_COOLDOWN_S:-840}"
|
||||
WAN_WATCH_COOLDOWN_FILE="${WAN_WATCH_COOLDOWN_FILE:-/var/run/wan-watch-deep.stamp}"
|
||||
WAN_ALERT_MS="${WAN_ALERT_MS:-50}"
|
||||
FE_ALERT_MS="${FE_ALERT_MS:-50}"
|
||||
BE_ALERT_MS="${BE_ALERT_MS:-50}"
|
||||
GW_ALERT_MS="${GW_ALERT_MS:-50}"
|
||||
ROUTER_ALERT_MS="${ROUTER_ALERT_MS:-50}"
|
||||
WAN_LOSS_ALERT_PCT="${WAN_LOSS_ALERT_PCT:-1}"
|
||||
WAN_MDEV_ALERT_MS="${WAN_MDEV_ALERT_MS:-80}"
|
||||
WAN_CONNTRACK_ALERT_PCT="${WAN_CONNTRACK_ALERT_PCT:-80}"
|
||||
WAN_PING_TARGET="${WAN_PING_TARGET:-1.1.1.1}"
|
||||
FE_PING_TARGET="${FE_PING_TARGET:-10.7.0.10}"
|
||||
BE_PING_TARGET="${BE_PING_TARGET-10.7.16.128}"
|
||||
ROUTER_PING_TARGET="${ROUTER_PING_TARGET:-none}"
|
||||
GW_PING_TARGET="${GW_PING_TARGET:-auto}"
|
||||
PING_COUNT="${PING_COUNT:-10}"
|
||||
PING_DEADLINE_S="${PING_DEADLINE_S:-15}"
|
||||
|
||||
_WAN_SNAPSHOT_DIR="$(CDPATH= cd -- "$(dirname "$0")" && pwd)"
|
||||
# shellcheck source=/dev/null
|
||||
. "$_WAN_SNAPSHOT_DIR/wan-deep-diag.sh" 2>/dev/null \
|
||||
|| . /usr/local/bin/wan-deep-diag.sh 2>/dev/null \
|
||||
|| . /usr/local/lib/wan-deep-diag.sh 2>/dev/null \
|
||||
|| true
|
||||
|
||||
usage() {
|
||||
sed -n '2,28p' "$0"
|
||||
printf '\nCommands:\n'
|
||||
printf ' (default) append one snapshot line to the log\n'
|
||||
printf ' --report summarize today vs yesterday\n'
|
||||
printf ' --tail [N] print last N log lines (default 24)\n'
|
||||
printf ' --correlate FILE… merge logs by 15-min slot (role= or path basename)\n'
|
||||
printf ' --alerts [N] recent kind=alert + kind=deep lines (default 40)\n'
|
||||
printf ' --help this text\n'
|
||||
}
|
||||
|
||||
need_ping() {
|
||||
command -v ping >/dev/null 2>&1 || {
|
||||
echo "wan-snapshot: missing ping" >&2
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
detect_role() {
|
||||
_role="$WAN_SNAPSHOT_ROLE"
|
||||
if [ "$_role" != "auto" ]; then
|
||||
printf '%s' "$_role"
|
||||
return 0
|
||||
fi
|
||||
_hn="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)"
|
||||
case "$_hn" in
|
||||
*raspberry*|*rpi*) printf 'pi' ;;
|
||||
*monstro*|*router*|*gw*) printf 'router' ;;
|
||||
*gentoo*|*wg0*|*fe*) printf 'fe' ;;
|
||||
*) printf 'host' ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ping_summary HOST -> loss_pct min_ms avg_ms max_ms mdev_ms (-1 for missing)
|
||||
ping_summary() {
|
||||
_host="$1"
|
||||
_out=""
|
||||
_out="$(ping -c "$PING_COUNT" -w "$PING_DEADLINE_S" "$_host" 2>/dev/null)" || true
|
||||
if [ -z "$_out" ]; then
|
||||
printf '%s %s %s %s %s' "-1" "-1" "-1" "-1" "-1"
|
||||
return 0
|
||||
fi
|
||||
_loss="$(printf '%s\n' "$_out" | awk '/packet loss/ {
|
||||
for (i = 1; i <= NF; i++) if ($i ~ /%$/) { gsub(/%/, "", $i); print $i; exit }
|
||||
}')"
|
||||
_min="-1"
|
||||
_avg="-1"
|
||||
_max="-1"
|
||||
_mdev="-1"
|
||||
_rtt="$(printf '%s\n' "$_out" | sed -n 's/.*= \([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\) ms.*/\1 \2 \3 \4/p' | head -1)"
|
||||
if [ -n "$_rtt" ]; then
|
||||
_min="${_rtt%% *}"
|
||||
_rest="${_rtt#* }"
|
||||
_avg="${_rest%% *}"
|
||||
_rest="${_rest#* }"
|
||||
_max="${_rest%% *}"
|
||||
_mdev="${_rest#* }"
|
||||
else
|
||||
_avg="$(printf '%s\n' "$_out" | awk -F'[=/ ]' '/min\/avg\/max/ { print $8; exit }')"
|
||||
[ -n "$_avg" ] || _avg="-1"
|
||||
fi
|
||||
[ -n "$_loss" ] || _loss="-1"
|
||||
printf '%s %s %s %s %s' "$_loss" "$_min" "$_avg" "$_max" "$_mdev"
|
||||
}
|
||||
|
||||
append_ping_fields() {
|
||||
_line="$1"
|
||||
_prefix="$2"
|
||||
_host="$3"
|
||||
_sum="$(ping_summary "$_host")"
|
||||
_loss="${_sum%% *}"
|
||||
_rest="${_sum#* }"
|
||||
_min="${_rest%% *}"
|
||||
_rest="${_rest#* }"
|
||||
_avg="${_rest%% *}"
|
||||
_rest="${_rest#* }"
|
||||
_max="${_rest%% *}"
|
||||
_mdev="${_rest#* }"
|
||||
printf '%s %s=%s %s_loss=%s%% %s_min_ms=%s %s_avg_ms=%s %s_max_ms=%s %s_mdev_ms=%s' \
|
||||
"$_line" "$_prefix" "$_host" "$_prefix" "$_loss" "$_prefix" "$_min" \
|
||||
"$_prefix" "$_avg" "$_prefix" "$_max" "$_prefix" "$_mdev"
|
||||
}
|
||||
|
||||
route_dev() {
|
||||
_host="$1"
|
||||
ip route get "$_host" 2>/dev/null | awk '{for (i = 1; i <= NF; i++) if ($i == "dev") { print $(i + 1); exit }}'
|
||||
}
|
||||
|
||||
route_gw() {
|
||||
_host="$1"
|
||||
ip route get "$_host" 2>/dev/null | awk '{for (i = 1; i <= NF; i++) if ($i == "via") { print $(i + 1); exit }}'
|
||||
}
|
||||
|
||||
default_gw() {
|
||||
ip route show default 2>/dev/null | awk '{print $3; exit}'
|
||||
}
|
||||
|
||||
load_1m() {
|
||||
awk '{print $1}' /proc/loadavg 2>/dev/null || echo "n/a"
|
||||
}
|
||||
|
||||
cpu_busy_pct() {
|
||||
awk '/^cpu / {
|
||||
idle = $5 + $6; total = 0
|
||||
for (i = 2; i <= NF; i++) total += $i
|
||||
if (total > 0) printf "%.0f", 100 * (total - idle) / total
|
||||
}' /proc/stat 2>/dev/null || echo "n/a"
|
||||
}
|
||||
|
||||
conntrack_pct() {
|
||||
_cur="$(cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null)" || _cur=""
|
||||
_max="$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null)" || _max=""
|
||||
if [ -n "$_cur" ] && [ -n "$_max" ] && [ "$_max" -gt 0 ] 2>/dev/null; then
|
||||
_pct=$((_cur * 100 / _max))
|
||||
printf 'conntrack=%s/%s(%s%%)' "$_cur" "$_max" "$_pct"
|
||||
fi
|
||||
}
|
||||
|
||||
neigh_state() {
|
||||
_host="$1"
|
||||
ip neigh show "$_host" 2>/dev/null | awk '{print $6; exit}'
|
||||
}
|
||||
|
||||
iface_drops_compact() {
|
||||
ip -s link 2>/dev/null | awk '
|
||||
/^[0-9]+:/ {
|
||||
if (iface != "" && iface !~ /^lo$/) {
|
||||
if (rx_drop + tx_drop + rx_err + tx_err > 0)
|
||||
printf "%s:rd=%d,td=%d,re=%d,te=%d ", iface, rx_drop, tx_drop, rx_err, tx_err
|
||||
}
|
||||
iface = $2; gsub(":", "", iface)
|
||||
rx_drop = tx_drop = rx_err = tx_err = 0
|
||||
}
|
||||
/RX:/ { getline; rx_err = $3; rx_drop = $4 }
|
||||
/TX:/ { getline; tx_err = $3; tx_drop = $4 }
|
||||
END {
|
||||
if (iface != "" && iface !~ /^lo$/ && rx_drop + tx_drop + rx_err + tx_err > 0)
|
||||
printf "%s:rd=%d,td=%d,re=%d,te=%d", iface, rx_drop, tx_drop, rx_err, tx_err
|
||||
}
|
||||
' | sed 's/ $//'
|
||||
}
|
||||
|
||||
# True if line is a snapshot row (legacy ts=… or kind=snapshot).
|
||||
is_snapshot_line() {
|
||||
case "$1" in
|
||||
kind=snapshot*) return 0 ;;
|
||||
ts=*) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
num_ge() {
|
||||
_a="$1"
|
||||
_b="$2"
|
||||
[ "$_a" != "-1" ] && [ -n "$_a" ] && awk -v a="$_a" -v b="$_b" 'BEGIN { exit (a + 0 >= b + 0) ? 0 : 1 }'
|
||||
}
|
||||
|
||||
watch_cooldown_active() {
|
||||
[ -r "$WAN_WATCH_COOLDOWN_FILE" ] || return 1
|
||||
_last="$(cat "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null)" || return 1
|
||||
_now="$(date +%s 2>/dev/null)" || return 1
|
||||
[ "$((_now - _last))" -lt "$WAN_WATCH_COOLDOWN_S" ] 2>/dev/null
|
||||
}
|
||||
|
||||
watch_cooldown_stamp() {
|
||||
date +%s > "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# Returns trigger list (comma-separated) or empty.
|
||||
detect_anomalies() {
|
||||
_line="$1"
|
||||
_role="$2"
|
||||
_triggers=""
|
||||
|
||||
for _pfx in wan fe be gw router; do
|
||||
_avg="$(field_value "$_line" "${_pfx}_avg_ms")"
|
||||
_loss="$(field_value "$_line" "${_pfx}_loss")"
|
||||
_mdev="$(field_value "$_line" "${_pfx}_mdev_ms")"
|
||||
_max="$(field_value "$_line" "${_pfx}_max_ms")"
|
||||
_thr="$FE_ALERT_MS"
|
||||
case "$_pfx" in
|
||||
wan) _thr="$WAN_ALERT_MS" ;;
|
||||
fe) _thr="$FE_ALERT_MS" ;;
|
||||
be) _thr="$BE_ALERT_MS" ;;
|
||||
gw) _thr="$GW_ALERT_MS" ;;
|
||||
router) _thr="$ROUTER_ALERT_MS" ;;
|
||||
esac
|
||||
if num_ge "$_avg" "$_thr"; then
|
||||
_triggers="${_triggers}${_triggers:+,}${_pfx}_avg=${_avg}"
|
||||
fi
|
||||
if num_ge "$_loss" "$WAN_LOSS_ALERT_PCT"; then
|
||||
_triggers="${_triggers}${_triggers:+,}${_pfx}_loss=${_loss}"
|
||||
fi
|
||||
if num_ge "$_mdev" "$WAN_MDEV_ALERT_MS"; then
|
||||
_triggers="${_triggers}${_triggers:+,}${_pfx}_mdev=${_mdev}"
|
||||
fi
|
||||
if num_ge "$_max" "$((_thr * 3))"; then
|
||||
_triggers="${_triggers}${_triggers:+,}${_pfx}_max=${_max}"
|
||||
fi
|
||||
done
|
||||
|
||||
_neigh="$(printf '%s\n' "$_line" | sed -n 's/.*neigh_fe=\([^ ]*\).*/\1/p')"
|
||||
if [ -n "$_neigh" ] && [ "$_neigh" != "REACHABLE" ] && [ "$_neigh" != "DELAY" ]; then
|
||||
_triggers="${_triggers}${_triggers:+,}neigh_fe=${_neigh}"
|
||||
fi
|
||||
|
||||
_ct_pct="$(printf '%s\n' "$_line" | sed -n 's/.*(\([0-9]*\)%).*/\1/p' | head -1)"
|
||||
if [ -n "$_ct_pct" ] && num_ge "$_ct_pct" "$WAN_CONNTRACK_ALERT_PCT"; then
|
||||
_triggers="${_triggers}${_triggers:+,}conntrack_pct=${_ct_pct}"
|
||||
fi
|
||||
|
||||
# Pi: WAN ok but FE slow -> path anomaly (even if FE just above threshold).
|
||||
if [ "$_role" = "pi" ]; then
|
||||
_wan_avg="$(field_value "$_line" "wan_avg_ms")"
|
||||
_fe_avg="$(field_value "$_line" "fe_avg_ms")"
|
||||
if num_ge "$_fe_avg" "$FE_ALERT_MS" && ! num_ge "$_wan_avg" "$WAN_ALERT_MS"; then
|
||||
case "$_triggers" in *fe_avg=*) ;; *)
|
||||
_triggers="${_triggers}${_triggers:+,}pi_fe_path" ;;
|
||||
esac
|
||||
fi
|
||||
fi
|
||||
|
||||
# Router: WAN bad, FE ok -> edge WAN leg (matches monstro xenbr0 pattern).
|
||||
if [ "$_role" = "router" ]; then
|
||||
_wan_avg="$(field_value "$_line" "wan_avg_ms")"
|
||||
_fe_avg="$(field_value "$_line" "fe_avg_ms")"
|
||||
if num_ge "$_wan_avg" "$WAN_ALERT_MS" && ! num_ge "$_fe_avg" "$FE_ALERT_MS"; then
|
||||
case "$_triggers" in *wan_avg=*) ;; *)
|
||||
_triggers="${_triggers}${_triggers:+,}router_wan_path" ;;
|
||||
esac
|
||||
fi
|
||||
fi
|
||||
|
||||
printf '%s' "$_triggers"
|
||||
}
|
||||
|
||||
maybe_deep_diag() {
|
||||
_line="$1"
|
||||
_ts="$2"
|
||||
_role="$3"
|
||||
_hostn="$4"
|
||||
|
||||
[ "$WAN_SNAPSHOT_WATCH" = "1" ] || return 0
|
||||
[ "${WAN_DEEP_DIAG_LOADED:-0}" = "1" ] || {
|
||||
echo "wan-snapshot: watch enabled but wan-deep-diag.sh not loaded (install alongside wan-snapshot.sh)" >&2
|
||||
return 0
|
||||
}
|
||||
|
||||
_triggers="$(detect_anomalies "$_line" "$_role")"
|
||||
[ -n "$_triggers" ] || return 0
|
||||
|
||||
if watch_cooldown_active; then
|
||||
wan_deep_emit alert "$_ts" "$_role" "$_hostn" \
|
||||
"parent_ts=$_ts triggers=\"$_triggers\" action=skipped_cooldown"
|
||||
return 0
|
||||
fi
|
||||
|
||||
_gw_target="$GW_PING_TARGET"
|
||||
if [ "$_gw_target" = "auto" ]; then
|
||||
_gw_target="$(default_gw)"
|
||||
fi
|
||||
_wdev="$(route_dev "$WAN_PING_TARGET")"
|
||||
_fdev="$(route_dev "$FE_PING_TARGET")"
|
||||
|
||||
wan_deep_diag_run "$_ts" "$_role" "$_hostn" "$_triggers" \
|
||||
"$WAN_PING_TARGET" "$FE_PING_TARGET" "$_gw_target" "$_wdev" "$_fdev"
|
||||
watch_cooldown_stamp
|
||||
}
|
||||
|
||||
append_watch_log() {
|
||||
_line="$1"
|
||||
_dir="$(dirname "$WAN_WATCH_LOG")"
|
||||
[ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true
|
||||
printf '%s\n' "$_line" >> "$WAN_WATCH_LOG"
|
||||
}
|
||||
|
||||
acquire_lock() {
|
||||
command -v flock >/dev/null 2>&1 || return 0
|
||||
_lock="$WAN_SNAPSHOT_LOCK"
|
||||
_dir="$(dirname "$_lock")"
|
||||
_fallback="${TMPDIR:-/tmp}/wan-snapshot.$(id -u 2>/dev/null || echo 0).lock"
|
||||
if [ ! -d "$_dir" ] || [ ! -w "$_dir" ] 2>/dev/null; then
|
||||
_lock="$_fallback"
|
||||
elif ! ( : >>"$_lock" ) 2>/dev/null; then
|
||||
_lock="$_fallback"
|
||||
fi
|
||||
exec 9>"$_lock" 2>/dev/null || return 0
|
||||
flock -n 9 || {
|
||||
echo "wan-snapshot: skip (already running, lock=$_lock)" >&2
|
||||
exit 0
|
||||
}
|
||||
}
|
||||
|
||||
snapshot_once() {
|
||||
need_ping
|
||||
acquire_lock
|
||||
|
||||
_ts="$(date -Iseconds 2>/dev/null || date '+%Y-%m-%dT%H:%M:%S%z')"
|
||||
_role="$(detect_role)"
|
||||
_host="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)"
|
||||
|
||||
_line="kind=snapshot ts=$_ts ver=2 role=$_role host=$_host"
|
||||
_line="$(append_ping_fields "$_line" "wan" "$WAN_PING_TARGET")"
|
||||
_line="$(append_ping_fields "$_line" "fe" "$FE_PING_TARGET")"
|
||||
|
||||
if [ -n "$BE_PING_TARGET" ] && [ "$BE_PING_TARGET" != "none" ]; then
|
||||
_line="$(append_ping_fields "$_line" "be" "$BE_PING_TARGET")"
|
||||
fi
|
||||
|
||||
if [ -n "$ROUTER_PING_TARGET" ] && [ "$ROUTER_PING_TARGET" != "none" ]; then
|
||||
_line="$(append_ping_fields "$_line" "router" "$ROUTER_PING_TARGET")"
|
||||
fi
|
||||
|
||||
_gw_target="$GW_PING_TARGET"
|
||||
if [ "$_gw_target" = "auto" ]; then
|
||||
_gw_target="$(default_gw)"
|
||||
[ -n "$_gw_target" ] || _gw_target="none"
|
||||
fi
|
||||
if [ -n "$_gw_target" ] && [ "$_gw_target" != "none" ]; then
|
||||
_line="$(append_ping_fields "$_line" "gw" "$_gw_target")"
|
||||
fi
|
||||
|
||||
if [ "$WAN_SNAPSHOT_EXTRAS" = "1" ]; then
|
||||
_line="$_line load1=$(load_1m) cpu_busy=$(cpu_busy_pct)%"
|
||||
_ct="$(conntrack_pct)"
|
||||
[ -n "$_ct" ] && _line="$_line $_ct"
|
||||
_wdev="$(route_dev "$WAN_PING_TARGET")"
|
||||
_fdev="$(route_dev "$FE_PING_TARGET")"
|
||||
[ -n "$_wdev" ] && _line="$_line route_wan=$_wdev"
|
||||
[ -n "$_fdev" ] && _line="$_line route_fe=$_fdev"
|
||||
_wgw="$(route_gw "$WAN_PING_TARGET")"
|
||||
[ -n "$_wgw" ] && _line="$_line via_wan=$_wgw"
|
||||
_nfe="$(neigh_state "$FE_PING_TARGET")"
|
||||
[ -n "$_nfe" ] && _line="$_line neigh_fe=$_nfe"
|
||||
_drops="$(iface_drops_compact)"
|
||||
[ -n "$_drops" ] && _line="$_line drops=\"$_drops\""
|
||||
fi
|
||||
|
||||
append_watch_log "$_line"
|
||||
if [ "$WAN_SNAPSHOT_LOG" != "$WAN_WATCH_LOG" ]; then
|
||||
_dir="$(dirname "$WAN_SNAPSHOT_LOG")"
|
||||
[ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true
|
||||
printf '%s\n' "$_line" >> "$WAN_SNAPSHOT_LOG"
|
||||
fi
|
||||
|
||||
maybe_deep_diag "$_line" "$_ts" "$_role" "$_host"
|
||||
|
||||
printf '%s\n' "$_line"
|
||||
}
|
||||
|
||||
# Extract numeric field from log line (supports wan_avg_ms and legacy layouts).
|
||||
field_value() {
|
||||
_line="$1"
|
||||
_field="$2"
|
||||
printf '%s\n' "$_line" | awk -v f="$_field" '
|
||||
{
|
||||
for (i = 1; i <= NF; i++) {
|
||||
if ($i ~ "^" f "=") {
|
||||
split($i, a, "=")
|
||||
v = a[2]
|
||||
gsub(/%/, "", v)
|
||||
if (v + 0 == v) { print v; exit }
|
||||
}
|
||||
}
|
||||
}
|
||||
'
|
||||
}
|
||||
|
||||
day_stats() {
|
||||
_log="$1"
|
||||
_date="$2"
|
||||
_field="$3"
|
||||
[ -r "$_log" ] || return 0
|
||||
awk -v d="$_date" -v f="$_field" '
|
||||
($0 ~ "^kind=snapshot ts=" d || ($0 ~ "^ts=" d && $0 !~ / kind=/)) {
|
||||
for (i = 1; i <= NF; i++) {
|
||||
if ($i ~ "^" f "=") {
|
||||
split($i, a, "=")
|
||||
v = a[2]
|
||||
gsub(/%/, "", v)
|
||||
if (v + 0 == v && v >= 0) {
|
||||
n++; s += v; if (v > max) max = v
|
||||
if (v >= 50) bad++
|
||||
vals[n] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
END {
|
||||
if (n == 0) { print "n/a n/a n/a n/a"; exit }
|
||||
asort(vals)
|
||||
p50 = vals[int((n + 1) * 0.5)]
|
||||
p90 = vals[int(n * 0.9)]
|
||||
if (p90 == "") p90 = vals[n]
|
||||
printf "%.2f %.2f %.2f %.0f %d/%d", s / n, p50, p90, max, bad + 0, n
|
||||
}
|
||||
' "$_log"
|
||||
}
|
||||
|
||||
day_max_loss() {
|
||||
_log="$1"
|
||||
_date="$2"
|
||||
_field="$3"
|
||||
[ -r "$_log" ] || return 0
|
||||
awk -v d="$_date" -v f="$_field" '
|
||||
($0 ~ "^kind=snapshot ts=" d || ($0 ~ "^ts=" d && $0 !~ / kind=/)) {
|
||||
for (i = 1; i <= NF; i++) {
|
||||
if ($i ~ "^" f "=") {
|
||||
split($i, a, "=")
|
||||
v = a[2]
|
||||
gsub(/%/, "", v)
|
||||
if (v + 0 == v && v >= 0 && v > max) max = v
|
||||
}
|
||||
}
|
||||
}
|
||||
END {
|
||||
if (max == "") print "n/a"
|
||||
else printf "%.0f", max
|
||||
}
|
||||
' "$_log"
|
||||
}
|
||||
|
||||
report_day_block() {
|
||||
_log="$1"
|
||||
_label="$2"
|
||||
_date="$3"
|
||||
_samples="$(grep -cE "^kind=snapshot ts=$_date|^ts=$_date" "$_log" 2>/dev/null || echo 0)"
|
||||
printf '%s=%s samples=%s\n' "$_label" "$_date" "$_samples"
|
||||
for _pfx in wan fe be router gw; do
|
||||
_stats="$(day_stats "$_log" "$_date" "${_pfx}_avg_ms")"
|
||||
_mean="${_stats%% *}"
|
||||
_rest="${_stats#* }"
|
||||
_p50="${_rest%% *}"
|
||||
_rest="${_rest#* }"
|
||||
_p90="${_rest%% *}"
|
||||
_rest="${_rest#* }"
|
||||
_max="${_rest%% *}"
|
||||
_bad="${_rest#* }"
|
||||
_loss="$(day_max_loss "$_log" "$_date" "${_pfx}_loss")"
|
||||
if [ "$_mean" != "n/a" ]; then
|
||||
printf ' %s: mean=%s p50=%s p90=%s max=%s bad(>=50ms)=%s loss_max=%s%%\n' \
|
||||
"$_pfx" "$_mean" "$_p50" "$_p90" "$_max" "$_bad" "$_loss"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
report() {
|
||||
_log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
|
||||
_today="$(date '+%Y-%m-%d')"
|
||||
_yesterday="$(date -d 'yesterday' '+%Y-%m-%d' 2>/dev/null || date -v-1d '+%Y-%m-%d' 2>/dev/null || echo '')"
|
||||
|
||||
if [ ! -r "$_log" ]; then
|
||||
echo "wan-snapshot: no log at $_log (run default command first)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
_role="$(detect_role)"
|
||||
printf 'log=%s role=%s\n' "$_log" "$_role"
|
||||
report_day_block "$_log" "today" "$_today"
|
||||
|
||||
if [ -n "$_yesterday" ] && grep -qE "^kind=snapshot ts=$_yesterday|^ts=$_yesterday" "$_log" 2>/dev/null; then
|
||||
report_day_block "$_log" "yesterday" "$_yesterday"
|
||||
fi
|
||||
|
||||
printf '\nInterpretation:\n'
|
||||
printf ' wan bad + fe good -> ISP/upstream or route_dev issue\n'
|
||||
printf ' wan + fe bad -> router/LAN/tunnel path\n'
|
||||
printf ' fe bad (pi) + fe ok (fe) -> path to FE, not FE host CPU\n'
|
||||
printf ' high mdev / max vs avg -> jitter (bufferbloat, conntrack, VPN)\n'
|
||||
printf ' conntrack>80%% -> edge table pressure (router)\n'
|
||||
printf ' neigh_fe not REACHABLE -> ARP/L2 flap toward FE\n'
|
||||
printf ' ping good, HTTPS slow -> DNAT/nginx above this probe\n'
|
||||
}
|
||||
|
||||
tail_log() {
|
||||
_n="${1:-24}"
|
||||
_log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
|
||||
[ -r "$_log" ] || {
|
||||
echo "wan-snapshot: no log at $_log" >&2
|
||||
exit 1
|
||||
}
|
||||
grep -E '^kind=snapshot |^ts=' "$_log" 2>/dev/null | tail -n "$_n"
|
||||
}
|
||||
|
||||
tail_alerts() {
|
||||
_n="${1:-40}"
|
||||
_log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
|
||||
[ -r "$_log" ] || {
|
||||
echo "wan-snapshot: no log at $_log" >&2
|
||||
exit 1
|
||||
}
|
||||
grep -E '^kind=alert |^kind=deep ' "$_log" 2>/dev/null | tail -n "$_n"
|
||||
}
|
||||
|
||||
# Guess role label for correlate when log line has no role=.
|
||||
log_role_label() {
|
||||
_path="$1"
|
||||
_line="$2"
|
||||
_role="$(field_value "$_line" "role")"
|
||||
if [ -n "$_role" ] && [ "$_role" != "0" ]; then
|
||||
printf '%s' "$_role"
|
||||
return 0
|
||||
fi
|
||||
case "$_path" in
|
||||
*raspberry*|*rpi*) printf 'pi' ;;
|
||||
*FE_gentoo*|*gentoo*) printf 'fe' ;;
|
||||
*monstro*|*router*) printf 'router' ;;
|
||||
*) printf '%s' "$(basename "$_path" .log)" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Bucket timestamp to 15-min wall-clock slot: YYYY-MM-DDTHH:00|15|30|45
|
||||
slot_key() {
|
||||
_ts="$1"
|
||||
_date="${_ts%%T*}"
|
||||
_rest="${_ts#*T}"
|
||||
_hour="${_rest%%:*}"
|
||||
_min="${_rest#*:}"
|
||||
_min="${_min%%:*}"
|
||||
_min="${_min%%+*}"
|
||||
_min="${_min%%-*}"
|
||||
_bucket=$(( (_min / 15) * 15 ))
|
||||
printf '%sT%s:%02d\n' "$_date" "$_hour" "$_bucket"
|
||||
}
|
||||
|
||||
correlate_logs() {
|
||||
if [ "$#" -lt 1 ]; then
|
||||
echo "wan-snapshot: --correlate needs at least one log file" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
_tmp="$(mktemp "${TMPDIR:-/tmp}/wan-correlate.XXXXXX")"
|
||||
# shellcheck disable=SC2064
|
||||
trap 'rm -f "$_tmp"' EXIT INT HUP
|
||||
|
||||
for _path in "$@"; do
|
||||
[ -r "$_path" ] || {
|
||||
echo "wan-snapshot: unreadable: $_path" >&2
|
||||
continue
|
||||
}
|
||||
_label="$(basename "$_path")"
|
||||
while IFS= read -r _line || [ -n "$_line" ]; do
|
||||
is_snapshot_line "$_line" || continue
|
||||
case "$_line" in
|
||||
kind=snapshot*) _ts="${_line#kind=snapshot ts=}" ;;
|
||||
ts=*) _ts="${_line#ts=}" ;;
|
||||
*) continue ;;
|
||||
esac
|
||||
_ts="${_ts%% *}"
|
||||
_slot="$(slot_key "$_ts")"
|
||||
_role="$(log_role_label "$_path" "$_line")"
|
||||
printf '%s\t%s\t%s\t%s\t%s\n' "$_slot" "$_role" "$_ts" "$_label" "$_line" >> "$_tmp"
|
||||
done < "$_path"
|
||||
done
|
||||
|
||||
printf 'slot\t'
|
||||
_roles="$(awk -F'\t' '{print $2}' "$_tmp" | sort -u | tr '\n' ' ')"
|
||||
for _role in $_roles; do
|
||||
printf '%s_wan\t%s_fe\t%s_be\t' "$_role" "$_role" "$_role"
|
||||
done
|
||||
printf '\n'
|
||||
|
||||
awk -F'\t' '
|
||||
{
|
||||
slot = $1; role = $2; ts = $3; line = $5
|
||||
key = slot SUBSEP role
|
||||
if (!(key in best_ts) || ts > best_ts[key]) {
|
||||
best_ts[key] = ts
|
||||
wan = fe = be = "-"
|
||||
for (i = 1; i <= split(line, a, " "); i++) {
|
||||
if (a[i] ~ /^wan_avg_ms=/) { split(a[i], b, "="); wan = b[2] }
|
||||
if (a[i] ~ /^fe_avg_ms=/) { split(a[i], b, "="); fe = b[2] }
|
||||
if (a[i] ~ /^be_avg_ms=/) { split(a[i], b, "="); be = b[2] }
|
||||
}
|
||||
W[slot, role, "wan"] = wan
|
||||
W[slot, role, "fe"] = fe
|
||||
W[slot, role, "be"] = be
|
||||
slots[slot] = 1
|
||||
roles[role] = 1
|
||||
}
|
||||
}
|
||||
END {
|
||||
nroles = 0
|
||||
for (r in roles) rolelist[++nroles] = r
|
||||
asort(rolelist)
|
||||
nslots = 0
|
||||
for (s in slots) slotlist[++nslots] = s
|
||||
asort(slotlist)
|
||||
for (si = 1; si <= nslots; si++) {
|
||||
s = slotlist[si]
|
||||
printf "%s", s
|
||||
for (ri = 1; ri <= nroles; ri++) {
|
||||
r = rolelist[ri]
|
||||
printf "\t%s\t%s\t%s", W[s, r, "wan"] + 0 ? W[s, r, "wan"] : "-", \
|
||||
W[s, r, "fe"] + 0 ? W[s, r, "fe"] : "-", \
|
||||
W[s, r, "be"] + 0 ? W[s, r, "be"] : "-"
|
||||
}
|
||||
printf "\n"
|
||||
}
|
||||
}
|
||||
' "$_tmp" | while IFS= read -r _row; do
|
||||
printf '%s\n' "$_row"
|
||||
done
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
-h|--help|help)
|
||||
usage
|
||||
;;
|
||||
--report|report)
|
||||
report
|
||||
;;
|
||||
--tail|tail)
|
||||
tail_log "${2:-24}"
|
||||
;;
|
||||
--alerts|alerts)
|
||||
tail_alerts "${2:-40}"
|
||||
;;
|
||||
--correlate|correlate)
|
||||
shift
|
||||
correlate_logs "$@"
|
||||
;;
|
||||
"")
|
||||
snapshot_once
|
||||
;;
|
||||
*)
|
||||
echo "wan-snapshot: unknown command: $1 (try --help)" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
# Backward-compatible wrapper — canonical script is wan-watch.sh in the same directory.
|
||||
D="$(CDPATH= cd -- "$(dirname "$0")" && pwd)"
|
||||
exec "$D/wan-watch.sh" "$@"
|
||||
|
||||
30
examples/network/wan-watch.env.example
Normal file
30
examples/network/wan-watch.env.example
Normal file
@@ -0,0 +1,30 @@
|
||||
# /etc/conf.d/wan-watch — source from cron on Pi, monstro, FE
|
||||
#
|
||||
# Deploy ONE file:
|
||||
# sudo cp wan-watch.sh /usr/local/bin/wan-watch.sh && sudo chmod +x /usr/local/bin/wan-watch.sh
|
||||
#
|
||||
# Cron (every 5 minutes):
|
||||
# */5 * * * * root . /etc/conf.d/wan-watch 2>/dev/null; /usr/local/bin/wan-watch.sh >>/var/log/wan-watch.cron.log 2>&1
|
||||
|
||||
WAN_WATCH_LOG=/var/log/wan-watch.log
|
||||
WAN_WATCH_ROLE=pi # pi | router | fe
|
||||
|
||||
# Pi
|
||||
# ROUTER_PING_TARGET=10.7.6.228
|
||||
# FE_PING_TARGET=10.7.0.10
|
||||
# BE_PING_TARGET=10.7.16.128
|
||||
|
||||
# monstro
|
||||
# WAN_WATCH_ROLE=router
|
||||
|
||||
# FE
|
||||
# WAN_WATCH_ROLE=fe
|
||||
# ROUTER_PING_TARGET=10.7.6.228
|
||||
|
||||
WAN_WATCH_DEEP=auto # auto | always | never
|
||||
WAN_WATCH_COOLDOWN_S=300 # 5 min between deep captures
|
||||
|
||||
# Thresholds (ms)
|
||||
# WAN_ALERT_MS=50
|
||||
# FE_ALERT_MS=50
|
||||
# CONNTRACK_ALERT_PCT=70
|
||||
671
examples/network/wan-watch.sh
Executable file
671
examples/network/wan-watch.sh
Executable file
@@ -0,0 +1,671 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# wan-watch.sh — one script, one log, immediate verdict every run.
|
||||
#
|
||||
# Deploy (Pi + monstro + FE):
|
||||
# sudo cp wan-watch.sh /usr/local/bin/wan-watch.sh
|
||||
# sudo chmod +x /usr/local/bin/wan-watch.sh
|
||||
# sudo cp wan-watch.env.example /etc/conf.d/wan-watch
|
||||
#
|
||||
# Cron (every 5 min — fast enough to catch 15-min pulses):
|
||||
# */5 * * * * root . /etc/conf.d/wan-watch 2>/dev/null; /usr/local/bin/wan-watch.sh >>/var/log/wan-watch.cron.log 2>&1
|
||||
#
|
||||
# Manual:
|
||||
# wan-watch.sh # cron snapshot + probes + verdict
|
||||
# wan-watch.sh --full # human-readable dump NOW (no wait)
|
||||
# wan-watch.sh --analyze # conclusion from log (last 24h default)
|
||||
# wan-watch.sh --timeline # hourly good/bad bars
|
||||
# wan-watch.sh --force-deep # full deep capture even if OK
|
||||
#
|
||||
set -eu
|
||||
|
||||
WAN_WATCH_LOG="${WAN_WATCH_LOG:-/var/log/wan-watch.log}"
|
||||
WAN_WATCH_ROLE="${WAN_WATCH_ROLE:-${WAN_SNAPSHOT_ROLE:-auto}}"
|
||||
WAN_WATCH_LOCK="${WAN_WATCH_LOCK:-/var/run/wan-watch.lock}"
|
||||
WAN_WATCH_COOLDOWN_S="${WAN_WATCH_COOLDOWN_S:-300}"
|
||||
WAN_WATCH_COOLDOWN_FILE="${WAN_WATCH_COOLDOWN_FILE:-/var/run/wan-watch-deep.stamp}"
|
||||
WAN_WATCH_DEEP="${WAN_WATCH_DEEP:-auto}"
|
||||
WAN_ANALYZE_HOURS="${WAN_ANALYZE_HOURS:-24}"
|
||||
|
||||
WAN_PING_TARGET="${WAN_PING_TARGET:-1.1.1.1}"
|
||||
FE_PING_TARGET="${FE_PING_TARGET:-10.7.0.10}"
|
||||
BE_PING_TARGET="${BE_PING_TARGET-10.7.16.128}"
|
||||
ROUTER_PING_TARGET="${ROUTER_PING_TARGET:-none}"
|
||||
GW_PING_TARGET="${GW_PING_TARGET:-auto}"
|
||||
|
||||
PING_COUNT="${PING_COUNT:-3}"
|
||||
PING_DEADLINE_S="${PING_DEADLINE_S:-6}"
|
||||
DEEP_PING_COUNT="${DEEP_PING_COUNT:-8}"
|
||||
DEEP_PING_DEADLINE_S="${DEEP_PING_DEADLINE_S:-10}"
|
||||
|
||||
WAN_ALERT_MS="${WAN_ALERT_MS:-50}"
|
||||
FE_ALERT_MS="${FE_ALERT_MS:-50}"
|
||||
BE_ALERT_MS="${BE_ALERT_MS:-50}"
|
||||
GW_ALERT_MS="${GW_ALERT_MS:-50}"
|
||||
ROUTER_ALERT_MS="${ROUTER_ALERT_MS:-50}"
|
||||
LOSS_ALERT_PCT="${LOSS_ALERT_PCT:-1}"
|
||||
MDEV_ALERT_MS="${MDEV_ALERT_MS:-80}"
|
||||
CONNTRACK_ALERT_PCT="${CONNTRACK_ALERT_PCT:-70}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
emit() {
|
||||
_kind="$1"; _ts="$2"; _role="$3"; _host="$4"
|
||||
shift 4
|
||||
_line="kind=$_kind ts=$_ts ver=3 role=$_role host=$_host $*"
|
||||
_dir="$(dirname "$WAN_WATCH_LOG")"
|
||||
mkdir -p "$_dir" 2>/dev/null || true
|
||||
printf '%s\n' "$_line" >> "$WAN_WATCH_LOG"
|
||||
}
|
||||
|
||||
detect_role() {
|
||||
_role="$WAN_WATCH_ROLE"
|
||||
if [ "$_role" != "auto" ]; then printf '%s' "$_role"; return; fi
|
||||
_hn="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)"
|
||||
case "$_hn" in
|
||||
*raspberry*|*rpi*) printf 'pi' ;;
|
||||
*monstro*|*router*|*gw*) printf 'router' ;;
|
||||
*gentoo*|*wg0*|*fe*) printf 'fe' ;;
|
||||
*) printf 'host' ;;
|
||||
esac
|
||||
}
|
||||
|
||||
default_gw() { ip route show default 2>/dev/null | awk '{print $3; exit}'; }
|
||||
|
||||
route_dev() {
|
||||
ip route get "$1" 2>/dev/null | awk '{for (i=1;i<=NF;i++) if ($i=="dev") {print $(i+1); exit}}'
|
||||
}
|
||||
|
||||
route_via() {
|
||||
ip route get "$1" 2>/dev/null | awk '{for (i=1;i<=NF;i++) if ($i=="via") {print $(i+1); exit}}'
|
||||
}
|
||||
|
||||
load_1m() { awk '{print $1}' /proc/loadavg 2>/dev/null || echo n/a; }
|
||||
|
||||
cpu_busy_pct() {
|
||||
awk '/^cpu /{idle=$5+$6;t=0;for(i=2;i<=NF;i++)t+=$i;if(t>0)printf"%.0f",100*(t-idle)/t}' /proc/stat 2>/dev/null || echo n/a
|
||||
}
|
||||
|
||||
conntrack_kv() {
|
||||
_c="$(cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null)" || _c=""
|
||||
_m="$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null)" || _m=""
|
||||
if [ -n "$_c" ] && [ -n "$_m" ] && [ "$_m" -gt 0 ] 2>/dev/null; then
|
||||
printf 'conntrack=%s/%s(%s%%)' "$_c" "$_m" "$((_c * 100 / _m))"
|
||||
fi
|
||||
}
|
||||
|
||||
neigh_state() { ip neigh show "$1" 2>/dev/null | awk '{print $6; exit}'; }
|
||||
|
||||
num_ge() {
|
||||
awk -v a="$1" -v b="$2" 'BEGIN{exit(a+0>=b+0)?0:1}' 2>/dev/null
|
||||
}
|
||||
|
||||
ping_stats() {
|
||||
_host="$1"; _cnt="${2:-$PING_COUNT}"; _dead="${3:-$PING_DEADLINE_S}"
|
||||
_out="$(ping -c "$_cnt" -w "$_dead" "$_host" 2>/dev/null)" || _out=""
|
||||
if [ -z "$_out" ]; then printf '%s %s %s %s %s' -1 -1 -1 -1 -1; return; fi
|
||||
_loss="$(printf '%s\n' "$_out" | awk '/packet loss/{for(i=1;i<=NF;i++) if($i~/%$/){gsub(/%/,"",$i);print $i;exit}}')"
|
||||
_rtt="$(printf '%s\n' "$_out" | sed -n 's/.*= \([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\) ms.*/\1 \2 \3 \4/p' | head -1)"
|
||||
if [ -n "$_rtt" ]; then
|
||||
set -- $_rtt
|
||||
printf '%s %s %s %s %s' "${_loss:-0}" "$1" "$2" "$3" "$4"
|
||||
else
|
||||
_avg="$(printf '%s\n' "$_out" | awk -F'[=/ ]' '/min\/avg\/max/{print $8;exit}')"
|
||||
printf '%s %s %s %s %s' "${_loss:-0}" "${_avg:--1}" "${_avg:--1}" "${_avg:--1}" 0
|
||||
fi
|
||||
}
|
||||
|
||||
append_ping() {
|
||||
_line="$1"; _pfx="$2"; _host="$3"
|
||||
set -- $(ping_stats "$_host")
|
||||
_loss="$1"; _min="$2"; _avg="$3"; _max="$4"; _mdev="$5"
|
||||
printf '%s %s=%s %s_loss=%s%% %s_min_ms=%s %s_avg_ms=%s %s_max_ms=%s %s_mdev_ms=%s' \
|
||||
"$_line" "$_pfx" "$_host" "$_pfx" "$_loss" "$_pfx" "$_min" "$_pfx" "$_avg" "$_pfx" "$_max" "$_pfx" "$_mdev"
|
||||
}
|
||||
|
||||
field_val() {
|
||||
printf '%s\n' "$2" | awk -v f="$1" '{for(i=1;i<=NF;i++) if($i~"^"f"="){split($i,a,"=");gsub(/%/,"",a[2]);print a[2];exit}}'
|
||||
}
|
||||
|
||||
acquire_lock() {
|
||||
command -v flock >/dev/null 2>&1 || return 0
|
||||
_lock="$WAN_WATCH_LOCK"
|
||||
_fb="${TMPDIR:-/tmp}/wan-watch.$$.lock"
|
||||
_dir="$(dirname "$_lock")"
|
||||
if [ ! -w "$_dir" ] 2>/dev/null; then _lock="${TMPDIR:-/tmp}/wan-watch.$(id -u 2>/dev/null || echo 0).lock"; fi
|
||||
exec 9>"$_lock" 2>/dev/null || return 0
|
||||
flock -n 9 || { echo "wan-watch: skip (running)" >&2; exit 0; }
|
||||
}
|
||||
|
||||
cooldown_active() {
|
||||
[ -r "$WAN_WATCH_COOLDOWN_FILE" ] || return 1
|
||||
_l="$(cat "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null)" || return 1
|
||||
_n="$(date +%s 2>/dev/null)" || return 1
|
||||
[ "$((_n - _l))" -lt "$WAN_WATCH_COOLDOWN_S" ] 2>/dev/null
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Probes: tunnels, services, kernel — cheap, every run
|
||||
# ---------------------------------------------------------------------------
|
||||
probe_emit() {
|
||||
_ts="$1"; _role="$2"; _host="$3"; _section="$4"
|
||||
shift 4
|
||||
emit probe "$_ts" "$_role" "$_host" "section=$_section $*"
|
||||
}
|
||||
|
||||
probe_all() {
|
||||
_ts="$1"; _role="$2"; _host="$3"
|
||||
|
||||
# Kernel modules (GRE/PPTP)
|
||||
if [ -r /proc/modules ]; then
|
||||
_gre="$(grep -E '^ip_gre |^gre ' /proc/modules 2>/dev/null | awk '{print $1}' | tr '\n' ',' | sed 's/,$//')"
|
||||
_pptp="$(grep -E 'nf_conntrack_pptp|nf_nat_pptp' /proc/modules 2>/dev/null | awk '{print $1}' | tr '\n' ',' | sed 's/,$//')"
|
||||
[ -n "$_gre" ] && probe_emit "$_ts" "$_role" "$_host" gre "loaded=${_gre}"
|
||||
[ -n "$_pptp" ] && probe_emit "$_ts" "$_role" "$_host" pptp_mod "loaded=${_pptp}"
|
||||
fi
|
||||
|
||||
# GRE / gretap links
|
||||
ip -d link show type gre 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" '
|
||||
/^[0-9]+:/{name=$2; gsub(":","",name); state=$0; getline; detail=$0;
|
||||
printf "kind=probe ts=%s ver=3 role=%s host=%s section=gre_iface name=%s up=%s\n", ts,role,host,name,(index(state,"UP")?"yes":"no")}
|
||||
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
|
||||
|
||||
# PPTP listeners / sessions
|
||||
_p1723="$(ss -lnpt 2>/dev/null | grep -c ':1723 ' || echo 0)"
|
||||
_pppd="$(pgrep -c pppd 2>/dev/null || echo 0)"
|
||||
_pptpd="$(pgrep -c pptpd 2>/dev/null || echo 0)"
|
||||
probe_emit "$_ts" "$_role" "$_host" pptp "listen1723=${_p1723} pppd=${_pppd} pptpd=${_pptpd}"
|
||||
|
||||
# tinc
|
||||
_tinc="$(pgrep -c tincd 2>/dev/null || echo 0)"
|
||||
if [ "$_tinc" -gt 0 ] 2>/dev/null; then
|
||||
_tn=""
|
||||
if command -v tincctl >/dev/null 2>&1; then
|
||||
for _net in /etc/tinc/*/tinc.conf; do
|
||||
[ -f "$_net" ] || continue
|
||||
_n="$(basename "$(dirname "$_net")")"
|
||||
_st="$(tincctl -n "$_n" status 2>/dev/null | tr '\n' ' ' | sed 's/ */ /g')"
|
||||
probe_emit "$_ts" "$_role" "$_host" tinc "net=${_n} tincd=${_tinc} status=${_st:-unknown}"
|
||||
done
|
||||
else
|
||||
probe_emit "$_ts" "$_role" "$_host" tinc "tincd=${_tinc} status=no_tincctl"
|
||||
fi
|
||||
else
|
||||
probe_emit "$_ts" "$_role" "$_host" tinc "tincd=0"
|
||||
fi
|
||||
|
||||
# WireGuard
|
||||
if command -v wg >/dev/null 2>&1; then
|
||||
wg show all 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" '
|
||||
/^interface:/ { iface=$2; peer=""; hs=""; ep=""; tx=""; rx="" }
|
||||
/^ public key:/ { pk=$3 }
|
||||
/^ endpoint:/ { ep=$2 }
|
||||
/^ latest handshake:/ { hs=$0; sub(/^ latest handshake: /,"",hs) }
|
||||
/^ transfer:/ { tx=$2; rx=$4 }
|
||||
/^$/ {
|
||||
if (iface != "")
|
||||
printf "kind=probe ts=%s ver=3 role=%s host=%s section=wireguard iface=%s endpoint=%s handshake=%s tx=%s rx=%s\n",
|
||||
ts, role, host, iface, ep, hs, tx, rx
|
||||
}
|
||||
END {
|
||||
if (iface != "")
|
||||
printf "kind=probe ts=%s ver=3 role=%s host=%s section=wireguard iface=%s endpoint=%s handshake=%s tx=%s rx=%s\n",
|
||||
ts, role, host, iface, ep, hs, tx, rx
|
||||
}
|
||||
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# iodine / dns tunnel iface
|
||||
_iod="$(pgrep -c iodined 2>/dev/null || echo 0)"
|
||||
probe_emit "$_ts" "$_role" "$_host" iodine "iodined=${_iod}"
|
||||
for _d in dns0 tun0 tap0; do
|
||||
ip link show "$_d" >/dev/null 2>&1 || continue
|
||||
ip -s link show "$_d" 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v d="$_d" '
|
||||
/RX:/{getline; re=$3; rd=$4} /TX:/{getline; te=$3; td=$4}
|
||||
END{printf "kind=probe ts=%s ver=3 role=%s host=%s section=tunnel_iface dev=%s rx_err=%s rx_drop=%s tx_err=%s tx_drop=%s\n",
|
||||
ts,role,host,d,re+0,rd+0,te+0,td+0}
|
||||
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
|
||||
done
|
||||
|
||||
# Named / squid / nginx (edge services)
|
||||
for _svc in named bind9 squid nginx iodined pptpd; do
|
||||
_c="$(pgrep -c "$_svc" 2>/dev/null || echo 0)"
|
||||
[ "$_c" -gt 0 ] 2>/dev/null || continue
|
||||
_cpu="$(ps -eo pcpu,comm 2>/dev/null | awk -v s="$_svc" '$2~s{print $1; exit}')"
|
||||
probe_emit "$_ts" "$_role" "$_host" service "name=${_svc} procs=${_c} cpu=${_cpu:-n/a}"
|
||||
done
|
||||
|
||||
# Key ifaces stats
|
||||
for _d in xenbr0 eno0 eth0 wlan0 br0; do
|
||||
ip link show "$_d" >/dev/null 2>&1 || continue
|
||||
ip -s link show "$_d" 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v d="$_d" '
|
||||
/RX:/{getline; re=$3; rd=$4} /TX:/{getline; te=$3; td=$4}
|
||||
END{printf "kind=probe ts=%s ver=3 role=%s host=%s section=iface dev=%s rx_drop=%s tx_drop=%s rx_err=%s tx_err=%s\n",
|
||||
ts,role,host,d,rd+0,td+0,re+0,te+0}
|
||||
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
|
||||
tc -s qdisc show dev "$_d" 2>/dev/null | head -3 | while IFS= read -r _ql; do
|
||||
probe_emit "$_ts" "$_role" "$_host" qdisc "dev=${_d} detail=${_ql}"
|
||||
done
|
||||
done
|
||||
|
||||
# softnet drops
|
||||
if [ -r /proc/net/softnet_stat ]; then
|
||||
awk -v ts="$_ts" -v role="$_role" -v host="$_host" '$2+0>0{
|
||||
printf "kind=probe ts=%s ver=3 role=%s host=%s section=softnet cpu=%d dropped=%s\n",ts,role,host,NR-1,$2
|
||||
}' /proc/net/softnet_stat >> "$WAN_WATCH_LOG" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# Routes + neigh for key targets
|
||||
for _label in wan:$WAN_PING_TARGET fe:$FE_PING_TARGET gw:${GW_PING_TARGET:-auto}; do
|
||||
_l="${_label%%:*}"; _t="${_label#*:}"
|
||||
[ "$_t" = "auto" ] && _t="$(default_gw)"
|
||||
[ -n "$_t" ] && [ "$_t" != none ] || continue
|
||||
ip route get "$_t" 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v l="$_l" -v t="$_t" '
|
||||
{dev=via=src=""; for(i=1;i<=NF;i++){if($i=="dev")dev=$(i+1);if($i=="via")via=$(i+1);if($i=="src")src=$(i+1)}
|
||||
printf "kind=probe ts=%s ver=3 role=%s host=%s section=route label=%s target=%s dev=%s via=%s src=%s\n",ts,role,host,l,t,dev,via,src}
|
||||
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
|
||||
done
|
||||
_ns="$(neigh_state "$FE_PING_TARGET")"
|
||||
[ -n "$_ns" ] && probe_emit "$_ts" "$_role" "$_host" neigh "ip=${FE_PING_TARGET} state=${_ns}"
|
||||
|
||||
# iptables hot rules (router/fe)
|
||||
if command -v iptables >/dev/null 2>&1 && [ "$_role" = "router" ]; then
|
||||
for _spec in filter:FORWARD nat:PREROUTING nat:POSTROUTING; do
|
||||
_tbl="${_spec%%:*}"; _ch="${_spec#*:}"
|
||||
iptables -t "$_tbl" -L "$_ch" -n -v --line-numbers 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v tbl="$_tbl" -v ch="$_ch" '
|
||||
/^[0-9]+/ && $2+0>1000 {
|
||||
printf "kind=probe ts=%s ver=3 role=%s host=%s section=iptables tbl=%s chain=%s line=%s pkts=%s target=%s\n",
|
||||
ts,role,host,tbl,ch,$1,$2,$4
|
||||
}' | head -6 >> "$WAN_WATCH_LOG" 2>/dev/null || true
|
||||
done
|
||||
fi
|
||||
|
||||
# Top CPU
|
||||
ps -eo pcpu=,pmem=,comm= --sort=-pcpu 2>/dev/null | head -5 | awk -v ts="$_ts" -v role="$_role" -v host="$_host" '
|
||||
NF>=3{printf "kind=probe ts=%s ver=3 role=%s host=%s section=proc rank=%d cpu=%s mem=%s comm=%s\n",ts,role,host,NR,$1,$2,$3}
|
||||
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
|
||||
|
||||
ss -s 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" '/TCP:|UDP:/{
|
||||
gsub(/^[ \t]+/,""); printf "kind=probe ts=%s ver=3 role=%s host=%s section=ss %s\n",ts,role,host,$0
|
||||
}' >> "$WAN_WATCH_LOG" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Deep capture (on anomaly)
|
||||
# ---------------------------------------------------------------------------
|
||||
deep_burst_line() {
|
||||
_ts="$1"; _role="$2"; _host="$3"; _label="$4"; _target="$5"
|
||||
set -- $(ping_stats "$_target" "$DEEP_PING_COUNT" "$DEEP_PING_DEADLINE_S")
|
||||
emit deep "$_ts" "$_role" "$_host" \
|
||||
"parent_ts=$_ts section=burst label=$_label target=$_target loss=$1% min_ms=$2 avg_ms=$3 max_ms=$4 mdev_ms=$5"
|
||||
}
|
||||
|
||||
deep_capture() {
|
||||
_ts="$1"; _role="$2"; _host="$3"; _triggers="$4"
|
||||
emit alert "$_ts" "$_role" "$_host" "parent_ts=$_ts triggers=\"$_triggers\""
|
||||
_ct="$(conntrack_kv)"
|
||||
emit deep "$_ts" "$_role" "$_host" "parent_ts=$_ts section=host load1=$(load_1m) cpu_busy=$(cpu_busy_pct)% mem_avail_kb=$(awk '/^MemAvailable:/{print $2}' /proc/meminfo 2>/dev/null || echo n/a) ${_ct}"
|
||||
deep_burst_line "$_ts" "$_role" "$_host" wan "$WAN_PING_TARGET"
|
||||
deep_burst_line "$_ts" "$_role" "$_host" fe "$FE_PING_TARGET"
|
||||
_gw="$GW_PING_TARGET"; [ "$_gw" = auto ] && _gw="$(default_gw)"
|
||||
[ -n "$_gw" ] && [ "$_gw" != none ] && deep_burst_line "$_ts" "$_role" "$_host" gw "$_gw"
|
||||
probe_all "$_ts" "$_role" "$_host"
|
||||
}
|
||||
|
||||
detect_triggers() {
|
||||
_line="$1"; _role="$2"; _t=""
|
||||
for _p in wan fe be gw router; do
|
||||
_a="$(field_val "${_p}_avg_ms" "$_line")"
|
||||
_l="$(field_val "${_p}_loss" "$_line")"
|
||||
_m="$(field_val "${_p}_mdev_ms" "$_line")"
|
||||
_thr="$FE_ALERT_MS"
|
||||
case "$_p" in wan) _thr="$WAN_ALERT_MS";; fe) _thr="$FE_ALERT_MS";;
|
||||
be) _thr="$BE_ALERT_MS";; gw) _thr="$GW_ALERT_MS";;
|
||||
router) _thr="$ROUTER_ALERT_MS";; esac
|
||||
num_ge "$_a" "$_thr" && _t="${_t}${_t:+,}${_p}_avg=${_a}"
|
||||
num_ge "$_l" "$LOSS_ALERT_PCT" && _t="${_t}${_t:+,}${_p}_loss=${_l}"
|
||||
num_ge "$_m" "$MDEV_ALERT_MS" && _t="${_t}${_t:+,}${_p}_mdev=${_m}"
|
||||
done
|
||||
_ct="$(printf '%s\n' "$_line" | sed -n 's/.*(\([0-9]*\)%).*/\1/p' | head -1)"
|
||||
num_ge "$_ct" "$CONNTRACK_ALERT_PCT" && _t="${_t}${_t:+,}conntrack_pct=${_ct}"
|
||||
_wan="$(field_val wan_avg_ms "$_line")"; _fe="$(field_val fe_avg_ms "$_line")"
|
||||
if [ "$_role" = pi ] && num_ge "$_fe" "$FE_ALERT_MS" && ! num_ge "$_wan" "$WAN_ALERT_MS"; then
|
||||
case "$_t" in *fe_avg=*) ;; *) _t="${_t}${_t:+,}pi_fe_path";; esac
|
||||
fi
|
||||
if [ "$_role" = router ] && num_ge "$_wan" "$WAN_ALERT_MS" && ! num_ge "$_fe" "$FE_ALERT_MS"; then
|
||||
case "$_t" in *wan_avg=*) ;; *) _t="${_t}${_t:+,}router_wan_path";; esac
|
||||
fi
|
||||
printf '%s' "$_t"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Verdict engine — runs EVERY time on current snapshot
|
||||
# ---------------------------------------------------------------------------
|
||||
compute_verdict() {
|
||||
_line="$1"; _role="$2"
|
||||
_probe_hint="${3:-}"
|
||||
_wan="$(field_val wan_avg_ms "$_line")"
|
||||
_fe="$(field_val fe_avg_ms "$_line")"
|
||||
_be="$(field_val be_avg_ms "$_line")"
|
||||
_gw="$(field_val gw_avg_ms "$_line")"
|
||||
_wmdev="$(field_val wan_mdev_ms "$_line")"
|
||||
_ct="$(printf '%s\n' "$_line" | sed -n 's/.*(\([0-9]*\)%).*/\1/p' | head -1)"
|
||||
|
||||
_verdict="OK"
|
||||
_conf="high"
|
||||
_summary="All probes within thresholds."
|
||||
_action="none"
|
||||
|
||||
if [ "$_role" = router ]; then
|
||||
if num_ge "$_wan" "$WAN_ALERT_MS" && ! num_ge "$_fe" 5; then
|
||||
_verdict="MONSTRO_WAN_BLOAT"
|
||||
_summary="WAN leg slow (~${_wan}ms) but FE local OK (~${_fe}ms). Edge xenbr0/upstream queue."
|
||||
_action="SQM/fq_codel on xenbr0; check gw $(route_via "$WAN_PING_TARGET" 2>/dev/null || echo 10.7.16.228); review squid/named load"
|
||||
elif num_ge "$_gw" "$GW_ALERT_MS" && num_ge "$_wan" "$WAN_ALERT_MS"; then
|
||||
_verdict="UPSTREAM_GW"
|
||||
_summary="WAN (${_wan}ms) and gw (${_gw}ms) both high — parent hop 10.7.16.228 stressed."
|
||||
_action="Check upstream link; correlate with ISP/gateway admin"
|
||||
elif num_ge "$_ct" "$CONNTRACK_ALERT_PCT"; then
|
||||
_verdict="CONNTRACK_PRESSURE"
|
||||
_summary="Conntrack at ${_ct}% — new flows may stall."
|
||||
_action="Inspect conntrack count; reduce NAT/tunnel sessions or raise nf_conntrack_max"
|
||||
elif num_ge "$_wmdev" "$MDEV_ALERT_MS"; then
|
||||
_verdict="WAN_JITTER"
|
||||
_summary="WAN jitter high (mdev=${_wmdev}ms) avg=${_wan}ms."
|
||||
_action="Bufferbloat likely; enable fq_codel"
|
||||
fi
|
||||
elif [ "$_role" = pi ]; then
|
||||
if num_ge "$_fe" "$FE_ALERT_MS" && ! num_ge "$_wan" "$WAN_ALERT_MS"; then
|
||||
_verdict="PATH_TO_FE"
|
||||
_summary="Pi ISP OK (~${_wan}ms) but FE path slow (~${_fe}ms). Problem at monstro edge, not Pi WiFi."
|
||||
_action="Check monstro wan-watch verdict; your traffic crosses modem WAN queue"
|
||||
elif num_ge "$_wan" "$WAN_ALERT_MS"; then
|
||||
_verdict="PI_ISP"
|
||||
_summary="Pi WAN to 1.1.1.1 slow (${_wan}ms)."
|
||||
_action="Local ISP/WiFi issue on Pi side"
|
||||
fi
|
||||
elif [ "$_role" = fe ]; then
|
||||
if num_ge "$_wan" "$WAN_ALERT_MS"; then
|
||||
_verdict="FE_WAN_EGRESS"
|
||||
_summary="FE WAN egress slow (${_wan}ms)."
|
||||
_action="Check FE default route and nginx upstream separately"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Probe hints from same run (grep log tail — set by caller in _probe_hint)
|
||||
case "${_probe_hint:-}" in
|
||||
*dns0*tx_drop=*|*tunnel_iface*dns0*)
|
||||
if [ "$_verdict" = "OK" ] || [ "$_verdict" = "MONSTRO_WAN_BLOAT" ]; then
|
||||
_verdict="TUNNEL_DNS0_LOAD"
|
||||
_summary="${_summary} iodine/dns0 tunnel drops elevated."
|
||||
_action="${_action}; inspect iodined and dns0 tx_drop"
|
||||
_conf="medium"
|
||||
fi ;;
|
||||
*named*cpu=*)
|
||||
if [ "$_verdict" = "MONSTRO_WAN_BLOAT" ] || [ "$_verdict" = "OK" ]; then
|
||||
_summary="${_summary} named DNS active on router."
|
||||
_conf="medium"
|
||||
fi ;;
|
||||
esac
|
||||
|
||||
printf 'verdict=%s confidence=%s summary="%s" action="%s" wan_ms=%s fe_ms=%s gw_ms=%s' \
|
||||
"$_verdict" "$_conf" "$_summary" "$_action" "$_wan" "$_fe" "$_gw"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main cron run
|
||||
# ---------------------------------------------------------------------------
|
||||
run_watch() {
|
||||
command -v ping >/dev/null 2>&1 || { echo "wan-watch: need ping" >&2; exit 1; }
|
||||
acquire_lock
|
||||
|
||||
_ts="$(date -Iseconds 2>/dev/null || date '+%Y-%m-%dT%H:%M:%S%z')"
|
||||
_role="$(detect_role)"
|
||||
_host="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)"
|
||||
|
||||
_gw="$GW_PING_TARGET"; [ "$_gw" = auto ] && _gw="$(default_gw)"
|
||||
_ct="$(conntrack_kv)"
|
||||
_wdev="$(route_dev "$WAN_PING_TARGET")"; _fdev="$(route_dev "$FE_PING_TARGET")"
|
||||
_wgw="$(route_via "$WAN_PING_TARGET")"
|
||||
_ns="$(neigh_state "$FE_PING_TARGET")"
|
||||
|
||||
_snap="$(append_ping "" wan "$WAN_PING_TARGET")"
|
||||
_snap="$(append_ping "$_snap" fe "$FE_PING_TARGET")"
|
||||
[ -n "$BE_PING_TARGET" ] && [ "$BE_PING_TARGET" != none ] && _snap="$(append_ping "$_snap" be "$BE_PING_TARGET")"
|
||||
[ -n "$ROUTER_PING_TARGET" ] && [ "$ROUTER_PING_TARGET" != none ] && _snap="$(append_ping "$_snap" router "$ROUTER_PING_TARGET")"
|
||||
[ -n "$_gw" ] && [ "$_gw" != none ] && _snap="$(append_ping "$_snap" gw "$_gw")"
|
||||
_snap="$_snap load1=$(load_1m) cpu_busy=$(cpu_busy_pct)%"
|
||||
[ -n "$_ct" ] && _snap="$_snap $_ct"
|
||||
[ -n "$_wdev" ] && _snap="$_snap route_wan=$_wdev"
|
||||
[ -n "$_fdev" ] && _snap="$_snap route_fe=$_fdev"
|
||||
[ -n "$_wgw" ] && _snap="$_snap via_wan=$_wgw"
|
||||
[ -n "$_ns" ] && _snap="$_snap neigh_fe=$_ns"
|
||||
|
||||
emit snapshot "$_ts" "$_role" "$_host" "$_snap" >/dev/null
|
||||
|
||||
probe_all "$_ts" "$_role" "$_host"
|
||||
|
||||
_probe_hint="$(tail -30 "$WAN_WATCH_LOG" 2>/dev/null | grep 'kind=probe' | tr '\n' ' ')"
|
||||
_verdict_kv="$(compute_verdict "$_snap" "$_role" "$_probe_hint")"
|
||||
_verdict_line="kind=verdict ts=$_ts ver=3 role=$_role host=$_host $_verdict_kv"
|
||||
printf '%s\n' "$_verdict_line" >> "$WAN_WATCH_LOG"
|
||||
|
||||
_triggers="$(detect_triggers "$_snap" "$_role")"
|
||||
_do_deep=0
|
||||
if [ -n "$_triggers" ]; then _do_deep=1; fi
|
||||
if [ "$WAN_WATCH_DEEP" = "always" ]; then _do_deep=1; fi
|
||||
if [ "$WAN_WATCH_DEEP" = "never" ]; then _do_deep=0; fi
|
||||
|
||||
if [ "$_do_deep" = 1 ]; then
|
||||
if cooldown_active && [ "$WAN_WATCH_DEEP" != "always" ]; then
|
||||
emit alert "$_ts" "$_role" "$_host" "parent_ts=$_ts triggers=\"$_triggers\" action=skipped_cooldown" >/dev/null
|
||||
else
|
||||
deep_capture "$_ts" "$_role" "$_host" "$_triggers"
|
||||
date +%s > "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
|
||||
# Human one-liner to cron log
|
||||
printf 'WAN-WATCH [%s/%s] %s\n' "$_role" "$_host" "$_verdict_kv"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# --analyze: final conclusion from log without waiting
|
||||
# ---------------------------------------------------------------------------
|
||||
analyze_log() {
|
||||
_log="$WAN_WATCH_LOG"
|
||||
_hours="${1:-$WAN_ANALYZE_HOURS}"
|
||||
[ -r "$_log" ] || { echo "wan-watch: no log $_log" >&2; exit 1; }
|
||||
|
||||
_cut="$(date -d "${_hours} hours ago" '+%Y-%m-%dT%H:%M' 2>/dev/null || echo "")"
|
||||
|
||||
echo "=== wan-watch analyze (last ${_hours}h) ==="
|
||||
echo "log=$_log"
|
||||
echo ""
|
||||
|
||||
awk -v cut="$_cut" '
|
||||
function ts_ok(t) { return (cut == "" || t >= cut) }
|
||||
/^kind=snapshot / && ts_ok($0) {
|
||||
for(i=1;i<=NF;i++) {
|
||||
if($i~/^wan_avg_ms=/){split($i,a,"="); wa=a[2]+0; wn++}
|
||||
if($i~/^fe_avg_ms=/){split($i,a,"="); fa=a[2]+0; fn++}
|
||||
if($i~/^gw_avg_ms=/){split($i,a,"="); ga=a[2]+0; gn++}
|
||||
if($i~/^role=/){split($i,a,"="); role=a[2]}
|
||||
}
|
||||
if(wa>=50){wb++} else if(wa>=0){wg++}
|
||||
if(fa>=50){fb++} else if(fa>=0){fg++}
|
||||
if(wa>=50 && fa<5) rw_ok_fe++ # router pattern
|
||||
if(fa>=50 && wa<30) pi_bad++ # pi pattern
|
||||
sn++
|
||||
}
|
||||
/^kind=verdict / && ts_ok($0) {
|
||||
if($0~/verdict=OK/) okv++; else badv++
|
||||
for(i=1;i<=NF;i++) if($i~/^verdict=/){split($i,a,"="); vc[a[2]]++}
|
||||
}
|
||||
/^kind=probe / && /section=tunnel_iface/ && /tx_drop=[1-9]/ { dns_drop++ }
|
||||
/^kind=probe / && /section=service/ && /name=named/ { named++ }
|
||||
/^kind=probe / && /section=service/ && /name=squid/ { squid++ }
|
||||
END {
|
||||
printf "snapshots=%d (wan_bad=%d wan_good=%d fe_bad=%d fe_good=%d)\n", sn, wb+0, wg+0, fb+0, fg+0
|
||||
printf "router_wan_bad_fe_local_ok=%d (MONSTRO_WAN_BLOAT signature)\n", rw_ok_fe+0
|
||||
printf "pi_fe_bad_wan_ok=%d (PATH_TO_FE signature)\n", pi_bad+0
|
||||
printf "verdicts: ok=%d anomaly=%d\n", okv+0, badv+0
|
||||
printf "verdict_counts:"
|
||||
for(v in vc) printf " %s=%d", v, vc[v]
|
||||
printf "\n"
|
||||
printf "probe_hints: dns0_tx_drop_events=%d named_samples=%d squid_samples=%d\n", dns_drop+0, named+0, squid+0
|
||||
pct = (sn>0) ? int(100*(wb+fb)/(2*sn)) : 0
|
||||
printf "\n"
|
||||
if(rw_ok_fe>=3 || pi_bad>=3) {
|
||||
printf "CONCLUSION: MONSTRO WAN EDGE QUEUEING (confidence=HIGH)\n"
|
||||
printf " Evidence: wan slow + fe local OK; Pi ISP stays ~9ms in prior data.\n"
|
||||
printf " Not: FE VM, BE, Pi WiFi, global ISP.\n"
|
||||
printf " Fix order: 1) fq_codel/SQM on xenbr0 2) gw 10.7.16.228 3) squid/named 4) iodine dns0\n"
|
||||
} else if(wb+fb==0) {
|
||||
printf "CONCLUSION: HEALTHY (confidence=HIGH)\n"
|
||||
} else {
|
||||
printf "CONCLUSION: MIXED/INTERMITTENT (%d%% bad samples) — keep */5 cron running\n", pct
|
||||
}
|
||||
}
|
||||
' "$_log"
|
||||
|
||||
echo ""
|
||||
echo "=== last 5 verdicts ==="
|
||||
grep '^kind=verdict ' "$_log" 2>/dev/null | tail -5 | while IFS= read -r _l; do
|
||||
_t="$(printf '%s\n' "$_l" | sed 's/.*ts=\([^ ]*\).*/\1/')"
|
||||
_v="$(printf '%s\n' "$_l" | sed 's/.*verdict=\([^ ]*\).*/\1/')"
|
||||
_s="$(printf '%s\n' "$_l" | sed 's/.*summary="\([^"]*\)".*/\1/')"
|
||||
printf ' %s %s %s\n' "$_t" "$_v" "$_s"
|
||||
done
|
||||
}
|
||||
|
||||
timeline_log() {
|
||||
_log="$WAN_WATCH_LOG"
|
||||
[ -r "$_log" ] || { echo "no log"; exit 1; }
|
||||
echo "=== hourly (snapshot wan/fe bad if avg>=50) ==="
|
||||
awk '
|
||||
/^kind=snapshot / {
|
||||
ts=$0; sub(/.*ts=/,"",ts); sub(/ .*/,"",ts)
|
||||
hour=substr(ts,1,13)
|
||||
for(i=1;i<=NF;i++){
|
||||
if($i~/^wan_avg_ms=/){split($i,a,"="); w=a[2]+0}
|
||||
if($i~/^fe_avg_ms=/){split($i,a,"="); f=a[2]+0}
|
||||
}
|
||||
t[hour]++; if(w>=50||f>=50) b[hour]++
|
||||
}
|
||||
END {
|
||||
for(h in t){ bh=b[h]+0; printf " %s %d/%d bad ", h, bh, t[h]
|
||||
for(i=0;i<t[h];i++) printf(i<bh?"#":".")
|
||||
printf "\n"
|
||||
}
|
||||
}
|
||||
' "$_log" | sort
|
||||
}
|
||||
|
||||
full_report() {
|
||||
_ts="$(date -Iseconds 2>/dev/null || date '+%Y-%m-%dT%H:%M:%S%z')"
|
||||
_role="$(detect_role)"
|
||||
_host="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)"
|
||||
echo "=== wan-watch FULL @ $_ts role=$_role host=$_host ==="
|
||||
PING_COUNT=5 PING_DEADLINE_S=10
|
||||
for _t in "$WAN_PING_TARGET" "$FE_PING_TARGET" "$BE_PING_TARGET" "$ROUTER_PING_TARGET" "$(default_gw)"; do
|
||||
[ -n "$_t" ] && [ "$_t" != none ] || continue
|
||||
echo "--- ping $_t ---"
|
||||
ping -c 5 -w 10 "$_t" 2>/dev/null || echo "fail"
|
||||
done
|
||||
echo "--- host ---"
|
||||
echo "load=$(load_1m) cpu=$(cpu_busy_pct)% $(conntrack_kv)"
|
||||
echo "--- probes ---"
|
||||
probe_all "$_ts" "$_role" "$_host"
|
||||
tail -40 "$WAN_WATCH_LOG" | grep "^kind=probe "
|
||||
echo "--- verdict ---"
|
||||
_snap="ts=$_ts role=$_role host=$_host"
|
||||
_snap="$(append_ping "$_snap" wan "$WAN_PING_TARGET")"
|
||||
_snap="$(append_ping "$_snap" fe "$FE_PING_TARGET")"
|
||||
compute_verdict "$_snap" "$_role"
|
||||
echo ""
|
||||
}
|
||||
|
||||
correlate_logs() {
|
||||
if [ "$#" -lt 1 ]; then echo "wan-watch: --correlate needs log files" >&2; exit 1; fi
|
||||
_tmp="$(mktemp "${TMPDIR:-/tmp}/wan-corr.XXXXXX")"
|
||||
trap 'rm -f "$_tmp"' EXIT INT HUP
|
||||
for _path in "$@"; do
|
||||
[ -r "$_path" ] || continue
|
||||
while IFS= read -r _line || [ -n "$_line" ]; do
|
||||
case "$_line" in
|
||||
kind=snapshot*) _ts="${_line#*ts=}"; _ts="${_ts%% *}" ;;
|
||||
ts=*) _ts="${_line#ts=}"; _ts="${_ts%% *}" ;;
|
||||
*) continue ;;
|
||||
esac
|
||||
_date="${_ts%%T*}"; _rest="${_ts#*T}"; _hour="${_rest%%:*}"
|
||||
_min="${_rest#*:}"; _min="${_min%%:*}"; _min="${_min%%+*}"; _min="${_min%%-*}"
|
||||
_slot="${_date}T${_hour}:$(printf '%02d' $((_min / 15 * 15)))"
|
||||
_role="$(printf '%s\n' "$_line" | awk '{for(i=1;i<=NF;i++) if($i~/^role=/){split($i,a,"=");print a[2];exit}}')"
|
||||
[ -n "$_role" ] || _role="$(basename "$_path" .log)"
|
||||
printf '%s\t%s\t%s\t%s\n' "$_slot" "$_role" "$_ts" "$_line" >> "$_tmp"
|
||||
done < "$_path"
|
||||
done
|
||||
printf 'slot\t'
|
||||
_roles="$(awk -F'\t' '{print $2}' "$_tmp" | sort -u | tr '\n' ' ')"
|
||||
for _r in $_roles; do printf '%s_wan\t%s_fe\t%s_verdict\t' "$_r" "$_r" "$_r"; done
|
||||
printf '\n'
|
||||
awk -F'\t' '
|
||||
{
|
||||
slot=$1; role=$2; ts=$3; line=$4; key=slot SUBSEP role
|
||||
if (!(key in best) || ts>best[key]) {
|
||||
best[key]=ts; wan=fe="-"; ver="-"
|
||||
for(i=1;i<=split(line,a," ");i++){
|
||||
if(a[i]~/^wan_avg_ms=/){split(a[i],b,"=");wan=b[2]}
|
||||
if(a[i]~/^fe_avg_ms=/){split(a[i],b,"=");fe=b[2]}
|
||||
}
|
||||
W[slot,role,"wan"]=wan; W[slot,role,"fe"]=fe
|
||||
slots[slot]=1; roles[role]=1
|
||||
}
|
||||
}
|
||||
END{
|
||||
nr=0; for(r in roles) rl[++nr]=r; asort(rl)
|
||||
ns=0; for(s in slots) sl[++ns]=s; asort(sl)
|
||||
for(i=1;i<=ns;i++){
|
||||
s=sl[i]; printf "%s", s
|
||||
for(j=1;j<=nr;j++){
|
||||
r=rl[j]; printf "\t%s\t%s\t-", W[s,r,"wan"]+0?W[s,r,"wan"]:"-", W[s,r,"fe"]+0?W[s,r,"fe"]:"-"
|
||||
}
|
||||
printf "\n"
|
||||
}
|
||||
}
|
||||
' "$_tmp"
|
||||
}
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
wan-watch.sh — unified network watchdog (ver=3)
|
||||
|
||||
Cron: */5 * * * * root . /etc/conf.d/wan-watch; /usr/local/bin/wan-watch.sh
|
||||
|
||||
Commands:
|
||||
(default) snapshot + probes + verdict (+ deep if bad)
|
||||
--full immediate human report
|
||||
--analyze [H] conclusion from log (default ${WAN_ANALYZE_HOURS}h)
|
||||
--timeline hourly bad/good bars
|
||||
--force-deep set WAN_WATCH_DEEP=always for one run
|
||||
--tail [N] last N snapshot+verdict lines
|
||||
--correlate FILE… merge Pi+router logs by 15-min slot
|
||||
--help
|
||||
|
||||
Log: $WAN_WATCH_LOG (kind=snapshot|probe|verdict|alert|deep)
|
||||
EOF
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
-h|--help|help) usage ;;
|
||||
--full|full) full_report ;;
|
||||
--analyze|analyze) analyze_log "${2:-$WAN_ANALYZE_HOURS}" ;;
|
||||
--timeline|timeline) timeline_log ;;
|
||||
--force-deep|force-deep) WAN_WATCH_DEEP=always run_watch ;;
|
||||
--tail|tail)
|
||||
_n="${2:-20}"
|
||||
grep -E '^kind=(snapshot|verdict|alert) ' "$WAN_WATCH_LOG" 2>/dev/null | tail -n "$_n" || true
|
||||
;;
|
||||
"") run_watch ;;
|
||||
*) echo "wan-watch: unknown $1"; usage; exit 1 ;;
|
||||
esac
|
||||
@@ -4,3 +4,5 @@ Defaults:nobody !requiretty
|
||||
Defaults:nginx !requiretty
|
||||
nobody ALL=(root) NOPASSWD: /usr/bin/wg set wg0 peer *
|
||||
nginx ALL=(root) NOPASSWD: /usr/bin/wg set wg0 peer *
|
||||
nginx ALL=(root) NOPASSWD: /usr/bin/wg show wg0 dump
|
||||
nobody ALL=(root) NOPASSWD: /usr/bin/wg show wg0 dump
|
||||
|
||||
@@ -28,7 +28,7 @@ $c["remote_access"] = array_merge($c["remote_access"] ?? [], [
|
||||
"wg_client_ip_prefix" => "172.200.2.",
|
||||
"wg_client_ip_min" => 10,
|
||||
"wg_client_ip_max" => 250,
|
||||
"wg_set_prefix" => "sudo ",
|
||||
"wg_set_prefix" => "sudo -n ",
|
||||
"provision_peers" => true,
|
||||
"require_wg_tools" => true,
|
||||
"session_ttl_s" => 3600,
|
||||
|
||||
Reference in New Issue
Block a user