From 86055bf509178c9c8f5b11a57110aa9932be5c66 Mon Sep 17 00:00:00 2001 From: Anton Afanasyeu Date: Sat, 13 Jun 2026 15:31:18 +0200 Subject: [PATCH] sync on be, vpn --- .../androidcast/dev/DevVpnStatusProbe.java | 60 +- .../RemoteAccessFailureReason.java | 2 +- .../RemoteAccessHeartbeatClient.java | 2 +- .../remoteaccess/RemoteAccessService.java | 7 +- .../remoteaccess/RemoteAccessStatusStore.java | 20 + docs/VPN_DEMO_GAPS_20260613.md | 28 + .../backend/public/assets/js/remote_access.js | 28 +- .../scripts/verify_remote_access_prod.sh | 10 +- .../backend/src/RemoteAccessRepository.php | 51 +- .../backend/src/WireGuardTrafficStats.php | 73 ++ .../crash_reporter/backend/src/bootstrap.php | 1 + .../crash_reporter/backend/views/layout.php | 1 + examples/network/README.md | 268 ++----- examples/network/wan-snapshot.sh | 698 +----------------- examples/network/wan-watch.env.example | 30 + examples/network/wan-watch.sh | 671 +++++++++++++++++ examples/wireguard/alpine-be-sudoers-wg | 2 + .../wireguard/install_be_remote_access.sh | 2 +- orchestration/sim/cluster0/ARCHITECTURE.md | 77 ++ orchestration/sim/cluster0/DEPLOY.md | 108 +++ orchestration/sim/cluster0/README.md | 17 + .../cluster0/bootstrap/cluster0-bootstrap.env | 5 + orchestration/sim/cluster0/cluster.env | 42 ++ orchestration/sim/cluster0/credentials.txt | 22 + orchestration/sim/cluster0/journal/README.md | 55 ++ .../sim/cluster0/mariadb/primary.cnf | 10 + .../sim/cluster0/mariadb/replica.cnf | 11 + .../sim/cluster0/nginx/apps-port80.conf | 119 +++ .../sim/cluster0/populate_lab_setup.sh | 136 ++++ .../sim/cluster0/scripts/baseline.sh | 67 ++ .../sim/cluster0/scripts/configure-nginx.sh | 12 + .../sim/cluster0/scripts/configure-php-fpm.sh | 13 + .../sim/cluster0/scripts/deploy-app.sh | 109 +++ .../scripts/journal-backfill-cluster0.sh | 57 ++ .../scripts/journal-rebuild-summary.sh | 35 + .../sim/cluster0/scripts/lib/common.sh | 192 +++++ .../sim/cluster0/scripts/load-schemas.sh | 39 + .../sim/cluster0/scripts/mariadb-primary.sh | 33 + .../sim/cluster0/scripts/mariadb-replica.sh | 73 ++ .../sim/cluster0/scripts/verify-cluster.sh | 34 + .../sim/cluster0/verify_global_setup.sh | 64 ++ .../sim/cluster0/verify_lab_setup.sh | 105 +++ 42 files changed, 2472 insertions(+), 917 deletions(-) create mode 100644 docs/VPN_DEMO_GAPS_20260613.md create mode 100644 examples/crash_reporter/backend/src/WireGuardTrafficStats.php create mode 100644 examples/network/wan-watch.env.example create mode 100755 examples/network/wan-watch.sh create mode 100644 orchestration/sim/cluster0/ARCHITECTURE.md create mode 100644 orchestration/sim/cluster0/DEPLOY.md create mode 100644 orchestration/sim/cluster0/README.md create mode 100644 orchestration/sim/cluster0/bootstrap/cluster0-bootstrap.env create mode 100644 orchestration/sim/cluster0/cluster.env create mode 100644 orchestration/sim/cluster0/credentials.txt create mode 100644 orchestration/sim/cluster0/journal/README.md create mode 100644 orchestration/sim/cluster0/mariadb/primary.cnf create mode 100644 orchestration/sim/cluster0/mariadb/replica.cnf create mode 100644 orchestration/sim/cluster0/nginx/apps-port80.conf create mode 100644 orchestration/sim/cluster0/populate_lab_setup.sh create mode 100644 orchestration/sim/cluster0/scripts/baseline.sh create mode 100644 orchestration/sim/cluster0/scripts/configure-nginx.sh create mode 100644 orchestration/sim/cluster0/scripts/configure-php-fpm.sh create mode 100644 orchestration/sim/cluster0/scripts/deploy-app.sh create mode 100644 orchestration/sim/cluster0/scripts/journal-backfill-cluster0.sh create mode 100644 orchestration/sim/cluster0/scripts/journal-rebuild-summary.sh create mode 100644 orchestration/sim/cluster0/scripts/lib/common.sh create mode 100644 orchestration/sim/cluster0/scripts/load-schemas.sh create mode 100644 orchestration/sim/cluster0/scripts/mariadb-primary.sh create mode 100644 orchestration/sim/cluster0/scripts/mariadb-replica.sh create mode 100644 orchestration/sim/cluster0/scripts/verify-cluster.sh create mode 100644 orchestration/sim/cluster0/verify_global_setup.sh create mode 100644 orchestration/sim/cluster0/verify_lab_setup.sh diff --git a/app/src/main/java/com/foxx/androidcast/dev/DevVpnStatusProbe.java b/app/src/main/java/com/foxx/androidcast/dev/DevVpnStatusProbe.java index b0a3c81..5bfa42b 100644 --- a/app/src/main/java/com/foxx/androidcast/dev/DevVpnStatusProbe.java +++ b/app/src/main/java/com/foxx/androidcast/dev/DevVpnStatusProbe.java @@ -57,19 +57,28 @@ public final class DevVpnStatusProbe { tunnelLevel = DevNetworkSelfTestReport.Level.PASS; } else if (RemoteAccessStatusStore.STATE_FAILURE.equals(tunnelState)) { tunnelLevel = DevNetworkSelfTestReport.Level.FAIL; + } else if (RemoteAccessStatusStore.STATE_WAIT.equals(last.state) + && isAwaitingWhitelist(last.detail)) { + tunnelLevel = DevNetworkSelfTestReport.Level.PASS; } else { tunnelLevel = DevNetworkSelfTestReport.Level.WARN; } - r.item(tunnelLevel, "tunnel-state", tunnelState); + String tunnelLabel = formatTunnelStateLabel(tunnelState, last); + r.item(tunnelLevel, "tunnel-state", tunnelLabel); if (mode != RemoteAccessMode.DISABLED && !RemoteAccessStatusStore.STATE_OFF.equals(controlPlane)) { - DevNetworkSelfTestReport.Level cpLevel = - RemoteAccessStatusStore.STATE_FAILURE.equals(controlPlane) - ? DevNetworkSelfTestReport.Level.FAIL - : RemoteAccessStatusStore.STATE_SUCCESS.equals(controlPlane) - ? DevNetworkSelfTestReport.Level.PASS - : DevNetworkSelfTestReport.Level.WARN; + DevNetworkSelfTestReport.Level cpLevel; + if (controlPlane.startsWith("ok (")) { + cpLevel = DevNetworkSelfTestReport.Level.PASS; + } else if (RemoteAccessStatusStore.STATE_FAILURE.equals(last.state) + && RemoteAccessFailureReason.isControlPlaneMethod(last.method)) { + cpLevel = DevNetworkSelfTestReport.Level.FAIL; + } else if (RemoteAccessStatusStore.STATE_SUCCESS.equals(last.state)) { + cpLevel = DevNetworkSelfTestReport.Level.PASS; + } else { + cpLevel = DevNetworkSelfTestReport.Level.WARN; + } r.item(cpLevel, "control-plane", controlPlane); } @@ -172,6 +181,18 @@ public final class DevVpnStatusProbe { } private static String resolveControlPlaneState(RemoteAccessStatusStore.Snapshot last) { + if (RemoteAccessStatusStore.STATE_WAIT.equals(last.state)) { + if (isAwaitingWhitelist(last.detail)) { + return "ok (awaiting whitelist)"; + } + if ("no_pending_session".equals(last.detail)) { + return "ok (awaiting operator session)"; + } + if ("rate_limited".equals(last.detail)) { + return "ok (poll rate limited)"; + } + return last.detail.isEmpty() ? RemoteAccessStatusStore.STATE_WAIT : last.detail; + } if (!RemoteAccessFailureReason.isControlPlaneMethod(last.method)) { if (RemoteAccessStatusStore.STATE_IDLE.equals(last.state) || RemoteAccessStatusStore.STATE_SUCCESS.equals(last.state)) { @@ -182,6 +203,22 @@ public final class DevVpnStatusProbe { return last.state; } + private static boolean isAwaitingWhitelist(String detail) { + return "not_whitelisted".equals(detail); + } + + private static String formatTunnelStateLabel(String tunnelState, RemoteAccessStatusStore.Snapshot last) { + if (RemoteAccessStatusStore.STATE_WAIT.equals(last.state)) { + if (isAwaitingWhitelist(last.detail)) { + return "idle (awaiting whitelist — no tunnel expected)"; + } + if ("no_pending_session".equals(last.detail)) { + return "idle (awaiting operator open session)"; + } + } + return tunnelState; + } + private static String resolveDisplayedFailReason( Context context, RemoteAccessMode mode, @@ -191,10 +228,12 @@ public final class DevVpnStatusProbe { if (mode == RemoteAccessMode.DISABLED) { return ""; } + if (!RemoteAccessStatusStore.STATE_FAILURE.equals(last.state)) { + return ""; + } String expanded = RemoteAccessFailureReason.expandForDisplay(context, last.method, last.failReason); - if (RemoteAccessFailureReason.isControlPlaneMethod(last.method) - && RemoteAccessStatusStore.STATE_FAILURE.equals(last.state)) { + if (RemoteAccessFailureReason.isControlPlaneMethod(last.method)) { if (!RemoteAccessStatusStore.STATE_SUCCESS.equals(tunnelState)) { return "[heartbeat] " + expanded; } @@ -204,8 +243,7 @@ public final class DevVpnStatusProbe { && RemoteAccessStatusStore.STATE_FAILURE.equals(tunnelState)) { return "[tunnel] " + expanded; } - if (RemoteAccessStatusStore.STATE_FAILURE.equals(controlPlane) - && !expanded.isEmpty()) { + if (!expanded.isEmpty()) { return "[heartbeat] " + expanded; } return ""; diff --git a/app/src/main/java/com/foxx/androidcast/remoteaccess/RemoteAccessFailureReason.java b/app/src/main/java/com/foxx/androidcast/remoteaccess/RemoteAccessFailureReason.java index 8572ee3..cf01638 100644 --- a/app/src/main/java/com/foxx/androidcast/remoteaccess/RemoteAccessFailureReason.java +++ b/app/src/main/java/com/foxx/androidcast/remoteaccess/RemoteAccessFailureReason.java @@ -14,7 +14,7 @@ public final class RemoteAccessFailureReason { public static String fromPollException(Context context, Exception e) { String host = heartbeatHost(context); if (e instanceof SocketTimeoutException) { - return "heartbeat HTTP timeout — device→" + host + " (15s connect/read)"; + return "heartbeat HTTP timeout — device→" + host + " (30s connect/read)"; } if (e instanceof UnknownHostException) { return "heartbeat DNS failed — " + host + " (" + safeMsg(e) + ")"; diff --git a/app/src/main/java/com/foxx/androidcast/remoteaccess/RemoteAccessHeartbeatClient.java b/app/src/main/java/com/foxx/androidcast/remoteaccess/RemoteAccessHeartbeatClient.java index 3e54ff3..6154a4d 100644 --- a/app/src/main/java/com/foxx/androidcast/remoteaccess/RemoteAccessHeartbeatClient.java +++ b/app/src/main/java/com/foxx/androidcast/remoteaccess/RemoteAccessHeartbeatClient.java @@ -18,7 +18,7 @@ import java.nio.charset.StandardCharsets; /** POST heartbeat type:ra to BE (shared by RemoteAccessService). */ public final class RemoteAccessHeartbeatClient { private static final String TAG = "RemoteAccessHB"; - private static final int HTTP_TIMEOUT_MS = 15_000; + private static final int HTTP_TIMEOUT_MS = 30_000; private RemoteAccessHeartbeatClient() {} diff --git a/app/src/main/java/com/foxx/androidcast/remoteaccess/RemoteAccessService.java b/app/src/main/java/com/foxx/androidcast/remoteaccess/RemoteAccessService.java index eaa43a6..29834ad 100644 --- a/app/src/main/java/com/foxx/androidcast/remoteaccess/RemoteAccessService.java +++ b/app/src/main/java/com/foxx/androidcast/remoteaccess/RemoteAccessService.java @@ -170,11 +170,12 @@ public final class RemoteAccessService extends Service { RemoteAccessStatusStore.INITIATOR_BE, raAction, resp.optString("session_id", "")); } else { + String waitReason = resp.optString("reason", ""); RemoteAccessStatusStore.record( - this, mode, RemoteAccessStatusStore.STATE_IDLE, + this, mode, RemoteAccessStatusStore.STATE_WAIT, "RemoteAccessService.runPoll", - RemoteAccessStatusStore.INITIATOR_DEVICE, - "", resp.optString("session_id", "")); + RemoteAccessStatusStore.INITIATOR_BE, + "", waitReason, resp.optString("session_id", "")); } } catch (Exception e) { Log.w(TAG, "poll failed: " + e.getMessage()); diff --git a/app/src/main/java/com/foxx/androidcast/remoteaccess/RemoteAccessStatusStore.java b/app/src/main/java/com/foxx/androidcast/remoteaccess/RemoteAccessStatusStore.java index 2aeb08b..0d2670e 100644 --- a/app/src/main/java/com/foxx/androidcast/remoteaccess/RemoteAccessStatusStore.java +++ b/app/src/main/java/com/foxx/androidcast/remoteaccess/RemoteAccessStatusStore.java @@ -20,12 +20,15 @@ public final class RemoteAccessStatusStore { public static final String STATE_FAILURE = "failure"; public static final String STATE_CONNECTING = "connecting"; public static final String STATE_IDLE = "idle"; + public static final String STATE_WAIT = "wait"; public static final String STATE_OFF = "off"; public static final String INITIATOR_DEVICE = "device"; public static final String INITIATOR_BE = "backend"; public static final String INITIATOR_USER = "user"; + private static final String KEY_DETAIL = "detail"; + private RemoteAccessStatusStore() {} public static void record( @@ -36,6 +39,18 @@ public final class RemoteAccessStatusStore { String initiator, String failReason, String sessionId) { + record(context, mode, state, method, initiator, failReason, "", sessionId); + } + + public static void record( + Context context, + RemoteAccessMode mode, + String state, + String method, + String initiator, + String failReason, + String detail, + String sessionId) { String type = mode == null || mode == RemoteAccessMode.DISABLED ? "off" : mode.name().toLowerCase(); prefs(context).edit() @@ -44,6 +59,7 @@ public final class RemoteAccessStatusStore { .putString(KEY_METHOD, method != null ? method : "") .putString(KEY_INITIATOR, initiator != null ? initiator : "") .putString(KEY_FAIL_REASON, failReason != null ? failReason : "") + .putString(KEY_DETAIL, detail != null ? detail : "") .putString(KEY_SESSION_ID, sessionId != null ? sessionId : "") .putLong(KEY_UPDATED_MS, System.currentTimeMillis()) .apply(); @@ -57,6 +73,7 @@ public final class RemoteAccessStatusStore { p.getString(KEY_METHOD, ""), p.getString(KEY_INITIATOR, ""), p.getString(KEY_FAIL_REASON, ""), + p.getString(KEY_DETAIL, ""), p.getString(KEY_SESSION_ID, ""), p.getLong(KEY_UPDATED_MS, 0L)); } @@ -71,6 +88,7 @@ public final class RemoteAccessStatusStore { public final String method; public final String initiator; public final String failReason; + public final String detail; public final String sessionId; public final long updatedMs; @@ -80,6 +98,7 @@ public final class RemoteAccessStatusStore { String method, String initiator, String failReason, + String detail, String sessionId, long updatedMs) { this.type = type != null ? type : "off"; @@ -87,6 +106,7 @@ public final class RemoteAccessStatusStore { this.method = method != null ? method : ""; this.initiator = initiator != null ? initiator : ""; this.failReason = failReason != null ? failReason : ""; + this.detail = detail != null ? detail : ""; this.sessionId = sessionId != null ? sessionId : ""; this.updatedMs = updatedMs; } diff --git a/docs/VPN_DEMO_GAPS_20260613.md b/docs/VPN_DEMO_GAPS_20260613.md new file mode 100644 index 0000000..3d9ed12 --- /dev/null +++ b/docs/VPN_DEMO_GAPS_20260613.md @@ -0,0 +1,28 @@ +# VPN / Remote Access demo — gap report (2026-06-13) + +See also [REMOTE_ACCESS_VALIDATION.md](../REMOTE_ACCESS_VALIDATION.md), [REMOTE_ACCESS_IMPL.md](../REMOTE_ACCESS_IMPL.md). + +## Fixed today (prod BE) + +- `config.php` **`remote_access` block** was missing — blocked real WireGuard connect credentials +- Installed via `examples/wireguard/install_be_remote_access.sh` +- **VPN traffic counters** in admin expanded device row (`WireGuardTrafficStats.php`) +- sudoers: `wg show wg0 dump` for nginx FPM user + +## Verified OK + +- `wg0` on BE, endpoint `ra.apps.f0xx.org:45340` +- UDP 45340 open on public IP +- Heartbeat disable/enable API +- Mobile boot restores RA mode (`RemoteAccessCoordinator.onBootIfNeeded`) + +## Open (PO / device) + +1. Full `ra_e2e_cli.sh` — needs admin login on laptop +2. UDP handshake + ping — needs phone or `ra_udp_debug.sh --apply` +3. SSH passthrough — RSSH track, not WG v1 +4. Doc port references still say 51820 in places; prod uses **45340** +5. Stale wg peer with `allowed ips: (none)` — manual cleanup +6. Persistent traffic history — live stats only today + +Journal: `/shared/journal/cluster0.journal.tsv` (tag `vpn-demo`). diff --git a/examples/crash_reporter/backend/public/assets/js/remote_access.js b/examples/crash_reporter/backend/public/assets/js/remote_access.js index b33b942..96de18f 100644 --- a/examples/crash_reporter/backend/public/assets/js/remote_access.js +++ b/examples/crash_reporter/backend/public/assets/js/remote_access.js @@ -81,6 +81,14 @@ return d.innerHTML; } + function formatBytes(n) { + const b = Number(n) || 0; + if (b < 1024) return b + ' B'; + if (b < 1024 * 1024) return (b / 1024).toFixed(1) + ' KiB'; + if (b < 1024 * 1024 * 1024) return (b / (1024 * 1024)).toFixed(2) + ' MiB'; + return (b / (1024 * 1024 * 1024)).toFixed(2) + ' GiB'; + } + function getColWidths() { try { const w = JSON.parse(lsGet(STORAGE.colWidths, '{}')); @@ -393,10 +401,26 @@ if (d.sdk_int != null) lines.push('Android API: ' + d.sdk_int); if (d.os_release) lines.push('Android version: ' + d.os_release); if (d.abis && d.abis.length) lines.push('ABIs: ' + formatAbis(d.abis)); - if (d.wan_ip) lines.push('WAN IP (last poll): ' + d.wan_ip); - if (d.lan_ip) lines.push('LAN IP: ' + d.lan_ip); + if (d.lan_ip) lines.push('Device LAN IP: ' + d.lan_ip); + if (d.device_wan_ip) lines.push('Device WAN IP: ' + d.device_wan_ip); + if (d.poll_source_ip) { + let pollLabel = 'Poll source IP (server view)'; + if (d.poll_source_is_private) { + pollLabel += ' — intra/private hop, not device WAN'; + } + lines.push(pollLabel + ': ' + d.poll_source_ip); + } if (d.vpn_ip) lines.push('VPN IP: ' + d.vpn_ip); if (d.vpn_public_key) lines.push('VPN public key: ' + d.vpn_public_key); + if (d.wg_rx_bytes != null || d.wg_tx_bytes != null) { + const rx = formatBytes(d.wg_rx_bytes || 0); + const tx = formatBytes(d.wg_tx_bytes || 0); + lines.push('VPN traffic (BE wg): ↓' + rx + ' ↑' + tx); + } + if (d.wg_latest_handshake > 0) { + lines.push('VPN last handshake: ' + new Date(d.wg_latest_handshake * 1000).toISOString()); + } + if (d.wg_endpoint) lines.push('VPN endpoint: ' + d.wg_endpoint); lines.push('App version: ' + (d.app_version || '—')); lines.push('Opt-in mode: ' + (d.opt_in_mode || 'none')); lines.push('Last seen: ' + (d.last_seen_at || '—')); diff --git a/examples/crash_reporter/backend/scripts/verify_remote_access_prod.sh b/examples/crash_reporter/backend/scripts/verify_remote_access_prod.sh index d97b7eb..769a45d 100755 --- a/examples/crash_reporter/backend/scripts/verify_remote_access_prod.sh +++ b/examples/crash_reporter/backend/scripts/verify_remote_access_prod.sh @@ -80,8 +80,14 @@ else note "php CLI cannot run wg genkey — ensure www-data can run wg if provision_peers=true" fi -# DB tables (via bootstrap) -"$PHP" -r ' +# DB tables (via bootstrap) — use FPM user + php81 so PDO/mysql extensions match production +DB_RUN="$PHP" +DB_USER="" +if [[ "$(id -u)" -eq 0 ]] && id nginx >/dev/null 2>&1; then + DB_RUN="sudo -u nginx $PHP" + DB_USER="nginx" +fi +$DB_RUN -r ' require "'"$ROOT"'/src/bootstrap.php"; RemoteAccessRepository::ensureSchema(); $pdo = Database::pdo(); diff --git a/examples/crash_reporter/backend/src/RemoteAccessRepository.php b/examples/crash_reporter/backend/src/RemoteAccessRepository.php index 9243308..95fea0c 100644 --- a/examples/crash_reporter/backend/src/RemoteAccessRepository.php +++ b/examples/crash_reporter/backend/src/RemoteAccessRepository.php @@ -266,11 +266,12 @@ final class RemoteAccessRepository { $pdo = Database::pdo(); $now = self::nowSql(); $pdo->prepare( - 'UPDATE remote_access_devices SET opt_in_mode = ?, last_random = ?, app_version = ?, last_seen_at = ?, updated_at = ? WHERE device_id = ?' + 'UPDATE remote_access_devices SET whitelisted = 0, opt_in_mode = ?, last_random = ?, app_version = ?, last_seen_at = ?, updated_at = ? WHERE device_id = ?' )->execute([self::OPT_NONE, $random, $appVersion, $now, $now, $deviceId]); self::closeActiveSessionsForDevice($deviceId, 'user_disabled'); self::logEvent($deviceId, null, null, 'user_disable', 'device_opt_out', null, $clientIp); + self::logEvent($deviceId, null, null, 'whitelist_revoked', 'device_opt_out', null, $clientIp); return self::raEnvelope('disabled', ['reason' => 'user_opt_out']); } @@ -667,7 +668,9 @@ final class RemoteAccessRepository { $sql .= ' WHERE company_id = ?'; $params[] = $companyId; } - $sql .= ' ORDER BY last_seen_at DESC, device_id ASC'; + $sql .= Database::isMysql() + ? ' ORDER BY last_seen_at DESC NULLS LAST, device_id ASC' + : ' ORDER BY datetime(COALESCE(last_seen_at, \'1970-01-01\')) DESC, device_id ASC'; $st = Database::pdo()->prepare($sql); $st->execute($params); return $st->fetchAll(PDO::FETCH_ASSOC) ?: []; @@ -845,11 +848,17 @@ final class RemoteAccessRepository { $row['os_release'] = $ctx['os_release'] ?? ''; $row['sdk_int'] = $ctx['sdk_int'] ?? null; $row['abis'] = $ctx['abis'] ?? []; - $row['wan_ip'] = self::resolveDeviceWanIp($row, $storedMeta); $row['lan_ip'] = self::resolveDeviceLanIp($storedMeta); + $row['poll_source_ip'] = self::resolvePollSourceIp($row); + $row['poll_source_is_private'] = self::isPrivateIp($row['poll_source_ip']); + $row['device_wan_ip'] = trim((string) ($storedMeta['wan_ip'] ?? '')); $net = self::lookupDeviceSessionNetwork($deviceId); $row['vpn_ip'] = $net['vpn_ip'] ?? ''; $row['vpn_public_key'] = $net['vpn_public_key'] ?? ''; + $row['wg_rx_bytes'] = $net['wg_rx_bytes'] ?? 0; + $row['wg_tx_bytes'] = $net['wg_tx_bytes'] ?? 0; + $row['wg_latest_handshake'] = $net['wg_latest_handshake'] ?? 0; + $row['wg_endpoint'] = $net['wg_endpoint'] ?? ''; $row['issue_count'] = $ctx['issue_count'] ?? 0; $row['graph_session_count'] = $ctx['graph_session_count'] ?? 0; $row['latest_issue_id'] = $ctx['latest_issue_id'] ?? null; @@ -911,12 +920,8 @@ final class RemoteAccessRepository { return (time() - $t) <= $maxAgeSec; } - /** @param array $storedMeta */ - private static function resolveDeviceWanIp(array $row, array $storedMeta): string { - $fromDevice = trim((string) ($storedMeta['wan_ip'] ?? '')); - if ($fromDevice !== '') { - return self::normalizeClientIp($fromDevice); - } + /** @param array $row */ + private static function resolvePollSourceIp(array $row): string { $fromRow = trim((string) ($row['last_client_ip'] ?? '')); if ($fromRow !== '') { return self::normalizeClientIp($fromRow); @@ -936,6 +941,17 @@ final class RemoteAccessRepository { return self::normalizeClientIp($ip); } + private static function isPrivateIp(string $ip): bool { + if ($ip === '' || !filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) { + return false; + } + if (!filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4 | FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE)) { + return true; + } + + return false; + } + /** @param array $storedMeta */ private static function resolveDeviceLanIp(array $storedMeta): string { return trim((string) ($storedMeta['lan_ip'] ?? '')); @@ -953,9 +969,16 @@ final class RemoteAccessRepository { return $raw; } - /** @return array{vpn_ip: string, vpn_public_key: string} */ + /** @return array{vpn_ip: string, vpn_public_key: string, wg_rx_bytes: int, wg_tx_bytes: int, wg_latest_handshake: int, wg_endpoint: string} */ private static function lookupDeviceSessionNetwork(string $deviceId): array { - $empty = ['vpn_ip' => '', 'vpn_public_key' => '']; + $empty = [ + 'vpn_ip' => '', + 'vpn_public_key' => '', + 'wg_rx_bytes' => 0, + 'wg_tx_bytes' => 0, + 'wg_latest_handshake' => 0, + 'wg_endpoint' => '', + ]; if ($deviceId === '') { return $empty; } @@ -978,9 +1001,15 @@ final class RemoteAccessRepository { $vpnIp = $addr !== '' ? preg_replace('/\/\d+$/', '', $addr) ?? $addr : ''; $pub = trim((string) ($row['wg_client_public_key'] ?? '')); + $stats = $pub !== '' ? WireGuardTrafficStats::forPublicKey($pub) : null; + return [ 'vpn_ip' => is_string($vpnIp) ? trim($vpnIp) : '', 'vpn_public_key' => $pub, + 'wg_rx_bytes' => (int) ($stats['rx_bytes'] ?? 0), + 'wg_tx_bytes' => (int) ($stats['tx_bytes'] ?? 0), + 'wg_latest_handshake' => (int) ($stats['latest_handshake'] ?? 0), + 'wg_endpoint' => (string) ($stats['endpoint'] ?? ''), ]; } diff --git a/examples/crash_reporter/backend/src/WireGuardTrafficStats.php b/examples/crash_reporter/backend/src/WireGuardTrafficStats.php new file mode 100644 index 0000000..8cf136f --- /dev/null +++ b/examples/crash_reporter/backend/src/WireGuardTrafficStats.php @@ -0,0 +1,73 @@ + */ + public static function peerMap(): array { + static $cache = null; + static $cacheAt = 0; + if ($cache !== null && (time() - $cacheAt) < 5) { + return $cache; + } + $cache = self::loadPeerMap(); + $cacheAt = time(); + return $cache; + } + + /** @return array{rx_bytes: int, tx_bytes: int, latest_handshake: int, endpoint: string}|null */ + public static function forPublicKey(string $publicKey): ?array { + $pub = trim($publicKey); + if ($pub === '') { + return null; + } + return self::peerMap()[$pub] ?? null; + } + + /** @return array */ + private static function loadPeerMap(): array { + $iface = (string) cfg('remote_access.wg_interface', 'wg0'); + $prefix = rtrim((string) cfg('remote_access.wg_set_prefix', ''), " \t"); + $cmd = ($prefix !== '' ? $prefix . ' ' : '') . 'wg show ' . escapeshellarg($iface) . ' dump 2>/dev/null'; + $raw = @shell_exec($cmd); + if (!is_string($raw) || trim($raw) === '') { + return []; + } + $out = []; + foreach (preg_split('/\r\n|\n|\r/', trim($raw)) as $line) { + if ($line === '' || str_starts_with($line, 'interface:')) { + continue; + } + $cols = explode("\t", $line); + if (count($cols) < 7) { + continue; + } + $pub = trim($cols[0]); + if ($pub === '') { + continue; + } + $out[$pub] = [ + 'rx_bytes' => (int) ($cols[5] ?? 0), + 'tx_bytes' => (int) ($cols[6] ?? 0), + 'latest_handshake' => (int) ($cols[4] ?? 0), + 'endpoint' => trim((string) ($cols[2] ?? '')), + ]; + } + return $out; + } + + public static function formatBytes(int $bytes): string { + if ($bytes < 1024) { + return $bytes . ' B'; + } + if ($bytes < 1024 * 1024) { + return round($bytes / 1024, 1) . ' KiB'; + } + if ($bytes < 1024 * 1024 * 1024) { + return round($bytes / (1024 * 1024), 2) . ' MiB'; + } + return round($bytes / (1024 * 1024 * 1024), 2) . ' GiB'; + } +} diff --git a/examples/crash_reporter/backend/src/bootstrap.php b/examples/crash_reporter/backend/src/bootstrap.php index 76e9e3a..876098a 100644 --- a/examples/crash_reporter/backend/src/bootstrap.php +++ b/examples/crash_reporter/backend/src/bootstrap.php @@ -68,6 +68,7 @@ require_once __DIR__ . '/UrlShortenerDatabase.php'; require_once __DIR__ . '/ShortLinksRepository.php'; require_once __DIR__ . '/WireGuardPeerProvisioner.php'; require_once __DIR__ . '/WireGuardAddressPool.php'; +require_once __DIR__ . '/WireGuardTrafficStats.php'; require_once __DIR__ . '/RsshSessionProvisioner.php'; require_once __DIR__ . '/AnalyticsHead.php'; diff --git a/examples/crash_reporter/backend/views/layout.php b/examples/crash_reporter/backend/views/layout.php index 7061866..efb6579 100644 --- a/examples/crash_reporter/backend/views/layout.php +++ b/examples/crash_reporter/backend/views/layout.php @@ -443,6 +443,7 @@ Devices appear after they poll remote access (dev settings → WireGuard or RSSH). Sort is by last seen — top rows are actively polling. Whitelist rows marked Needs whitelist (opted in, not yet allowed). Stale rows have not polled in 7+ days. + Poll source IP is the address BE sees on the HTTP request (often a router or proxy on 10.7.x.x) — not the device WAN or LAN.

diff --git a/examples/network/README.md b/examples/network/README.md index 285c855..d60c107 100644 --- a/examples/network/README.md +++ b/examples/network/README.md @@ -1,204 +1,92 @@ -# Edge network probes +# Edge network watchdog -Scripts for **router**, **FE**, and **external vantage** (e.g. Raspberry Pi) — not BE app hosts. +**One script:** `wan-watch.sh` — deploy everywhere, read one log, get a verdict every 5 minutes. -## Auto watchdog (v2.1) - -When a cron snapshot exceeds thresholds, `wan-snapshot.sh` appends **alert** + **deep** lines to the same log (`WAN_WATCH_LOG`). - -| `kind=` | Meaning | -|---------|---------| -| `snapshot` | Normal 15-min ping row | -| `alert` | Threshold breached; lists `triggers=` | -| `deep` | One parseable fact (`section=host`, `burst`, `iface`, `iptables`, …) | - -**Deploy both files:** +## Deploy ```bash -sudo cp examples/network/wan-snapshot.sh examples/network/wan-deep-diag.sh /usr/local/bin/ -sudo chmod +x /usr/local/bin/wan-snapshot.sh /usr/local/bin/wan-deep-diag.sh +sudo cp examples/network/wan-watch.sh /usr/local/bin/wan-watch.sh +sudo chmod +x /usr/local/bin/wan-watch.sh +sudo cp examples/network/wan-watch.env.example /etc/conf.d/wan-watch +# edit WAN_WATCH_ROLE=pi|router|fe ``` -**Env** (`/etc/conf.d/wan-snapshot`): +Remove old cron entries for `wan-snapshot.sh` / `monstro-netdiag.sh`. **One cron only:** + +```cron +*/5 * * * * root . /etc/conf.d/wan-watch 2>/dev/null; /usr/local/bin/wan-watch.sh >>/var/log/wan-watch.cron.log 2>&1 +``` + +Log: `/var/log/wan-watch.log` + +## Every run (5 min) captures + +| Layer | What | +|-------|------| +| **Ping matrix** | wan, fe, be, gw, router — loss/min/avg/max/mdev | +| **Verdict** | `kind=verdict` — immediate conclusion + action hint | +| **Tunnels** | GRE links/modules, PPTP (1723/pppd/pptpd), **tinc** (tincctl status), **WireGuard**, **iodine/dns0/tun0** | +| **Edge services** | named, squid, nginx, iodined — proc count + CPU | +| **Host** | load, cpu, conntrack, softnet drops, ss summary | +| **Network** | routes, neigh, iface drops, qdisc backlog, iptables hot rules (router) | +| **On anomaly** | `kind=alert` + `kind=deep` burst pings + full probe repeat (5 min cooldown) | + +## Commands + +```bash +wan-watch.sh # cron default +wan-watch.sh --full # instant human dump (when slow NOW) +wan-watch.sh --analyze # final conclusion from log (last 24h) — no waiting +wan-watch.sh --analyze 6 # last 6 hours +wan-watch.sh --timeline # hourly good/bad bars +wan-watch.sh --correlate /var/log/wan-watch.log path/to/other/log +wan-watch.sh --force-deep # full capture even if OK +wan-watch.sh --tail 30 +``` + +## Verdicts (automatic) + +| verdict | Meaning | +|---------|---------| +| `OK` | Within thresholds | +| `MONSTRO_WAN_BLOAT` | Router WAN slow, FE local OK — **your main issue** | +| `UPSTREAM_GW` | WAN + gw 10.7.16.228 both slow | +| `PATH_TO_FE` | Pi ISP OK, path to 10.7.0.10 slow | +| `CONNTRACK_PRESSURE` | nf_conntrack > 70% | +| `TUNNEL_DNS0_LOAD` | iodine dns0 drops elevated | +| `WAN_JITTER` | High mdev, bufferbloat signal | + +Cron stdout example: + +```text +WAN-WATCH [router/monstro] verdict=MONSTRO_WAN_BLOAT confidence=high summary="WAN leg slow..." action="SQM/fq_codel on xenbr0..." +``` + +## `--analyze` example output + +```text +CONCLUSION: MONSTRO WAN EDGE QUEUEING (confidence=HIGH) + Fix order: 1) fq_codel/SQM on xenbr0 2) gw 10.7.16.228 3) squid/named 4) iodine dns0 +``` + +Run this **any time** — no need to wait 24h after deploy. + +## Host config (`/etc/conf.d/wan-watch`) ```sh WAN_WATCH_LOG=/var/log/wan-watch.log -WAN_SNAPSHOT_LOG=/var/log/wan-watch.log -WAN_SNAPSHOT_WATCH=1 +WAN_WATCH_ROLE=pi # or router on monstro +ROUTER_PING_TARGET=10.7.6.228 # Pi only +WAN_WATCH_DEEP=auto +WAN_WATCH_COOLDOWN_S=300 ``` -**Triggers (defaults):** `*_avg_ms >= 50`, `*_loss >= 1%`, `mdev >= 80`, `conntrack >= 80%`, `neigh_fe` not REACHABLE/DELAY, plus role hints `pi_fe_path` / `router_wan_path`. +## Legacy -**Cooldown:** one deep capture per 14 min (`WAN_WATCH_COOLDOWN_S=840`) to avoid cron pile-up. +- `wan-snapshot.sh` → wrapper to `wan-watch.sh` +- `wan-deep-diag.sh` → merged into `wan-watch.sh` +- `monstro-netdiag.sh` → use `wan-watch.sh --full` instead -**Review after a bad spell:** +## Known conclusion (from your logs) -```bash -/usr/local/bin/wan-snapshot.sh --alerts 80 -/usr/local/bin/wan-snapshot.sh --correlate /var/log/wan-watch.log … -grep 'kind=deep.*section=burst' /var/log/wan-watch.log | tail -grep 'kind=deep.*section=iptables' /var/log/wan-watch.log | tail -``` - -Example deep lines: - -```text -kind=alert ts=2026-06-12T21:45:01+03:00 role=router host=monstro parent_ts=... triggers="wan_avg=535.365,router_wan_path" -kind=deep ts=2026-06-12T21:45:01+03:00 role=router host=monstro parent_ts=... section=burst label=wan target=1.1.1.1 loss=0% avg_ms=520 max_ms=790 mdev_ms=110 -kind=deep ts=2026-06-12T21:45:01+03:00 role=router host=monstro parent_ts=... section=iptables tbl=filter chain=FORWARD line=3 pkts=188000 target=ACCEPT -``` - -## `wan-snapshot.sh` (v2) - -Periodic ping comparison plus lightweight host/path context. - -| Target | Default | Meaning | -|--------|---------|---------| -| **WAN** | `1.1.1.1` | ISP/upstream leg | -| **FE** | `10.7.0.10` | Path to Gentoo FE | -| **BE** | `10.7.16.128` | Deeper LAN (disable: `BE_PING_TARGET=none`) | -| **router** | `none` | Optional modem ping (`ROUTER_PING_TARGET` on Pi) | -| **gw** | auto | Default gateway ping | - -Each target logs: `loss`, `min`, `avg`, `max`, `mdev` (jitter). - -Extras (`WAN_SNAPSHOT_EXTRAS=1`): `load1`, `cpu_busy`, `conntrack`, `route_wan`/`route_fe`, `via_wan`, `neigh_fe`, non-zero `drops`. - -Log format: `ver=2 role=pi|fe|router host=…` — older lines without `ver=` still work in `--report` / `--correlate`. - -### Install (any host) - -```bash -sudo cp examples/network/wan-snapshot.sh /usr/local/bin/wan-snapshot.sh -sudo chmod +x /usr/local/bin/wan-snapshot.sh -sudo cp examples/network/wan-snapshot.env.example /etc/conf.d/wan-snapshot -# edit /etc/conf.d/wan-snapshot for role-specific targets -``` - -### Cron (every 15 minutes — **one** entry per host) - -```cron -*/15 * * * * root . /etc/conf.d/wan-snapshot 2>/dev/null; /usr/local/bin/wan-snapshot.sh >>/var/log/wan-quality.cron.log 2>&1 -``` - -The script uses `flock` on `/var/run/wan-snapshot.lock` so duplicate crontab lines do not double-log. - -Log: `/var/log/wan-quality.log` (override with `WAN_SNAPSHOT_LOG`). - -### Suggested `/etc/conf.d/wan-snapshot` - -**Raspberry Pi** (path into infra matters most): - -```sh -WAN_SNAPSHOT_ROLE=pi -ROUTER_PING_TARGET=10.7.6.228 -FE_PING_TARGET=10.7.0.10 -BE_PING_TARGET=10.7.16.128 -``` - -**Gentoo FE**: - -```sh -WAN_SNAPSHOT_ROLE=fe -FE_PING_TARGET=10.7.0.10 -BE_PING_TARGET=10.7.16.128 -ROUTER_PING_TARGET=10.7.6.228 -``` - -**Debian router**: - -```sh -WAN_SNAPSHOT_ROLE=router -FE_PING_TARGET=10.7.0.10 -BE_PING_TARGET=10.7.16.128 -``` - -### Usage - -```bash -# one line appended + printed -sudo /usr/local/bin/wan-snapshot.sh - -# today vs yesterday (mean, p50, p90, bad>=50ms counts) -/usr/local/bin/wan-snapshot.sh --report - -# recent samples -/usr/local/bin/wan-snapshot.sh --tail 48 - -# merge Pi + FE (+ router) logs by 15-min slot -/usr/local/bin/wan-snapshot.sh --correlate \ - tmp/raspberrypi/var/log/wan-quality.log \ - tmp/FE_gentoo/var/log/wan-quality.log -``` - -Example v2 line: - -```text -ts=2026-06-12T15:30:01+03:00 ver=2 role=fe host=wg0 wan=1.1.1.1 wan_loss=0% wan_min_ms=9.1 wan_avg_ms=10.6 wan_max_ms=12.4 wan_mdev_ms=0.8 fe=10.7.0.10 fe_loss=0% fe_min_ms=0.05 fe_avg_ms=0.07 fe_max_ms=0.10 fe_mdev_ms=0.01 be=10.7.16.128 be_loss=0% be_avg_ms=0.20 load1=0.12 cpu_busy=4% route_wan=eno0 route_fe=lo neigh_fe=REACHABLE -``` - -### Reading results - -| Pattern | Likely cause | -|---------|----------------| -| `wan` bad, `fe` good (on router) | ISP/upstream | -| `wan` + `fe` bad on router | modem CPU, conntrack, LAN | -| `fe` bad on **pi**, `fe` good on **fe** | path/tunnel to FE, not FE VM | -| `wan_max_ms` >> `wan_avg_ms` or high `mdev` | jitter / bufferbloat | -| `conntrack>80%` | edge table full | -| `neigh_fe` not `REACHABLE` | ARP/L2 toward FE | -| `route_fe=tun0` / `ppp0` on Pi | metric is tunnel quality | -| ping good, HTTPS slow | DNAT/nginx (not measured here) | - -From **outside** LAN, still run occasionally: - -```bash -mtr -rwzc 50 134.17.26.161 -``` - -## `monstro-netdiag.sh` (bottleneck + iptables + processes) - -Run on the **Debian edge router** (`10.7.6.228`) when infra feels stuck. Complements `wan-snapshot` with on-demand depth. - -```bash -sudo cp examples/network/monstro-netdiag.sh /usr/local/bin/ -sudo chmod +x /usr/local/bin/monstro-netdiag.sh -``` - -**Cron** (optional; pair with wan-snapshot on router only): - -```cron -*/15 * * * * root /usr/local/bin/monstro-netdiag.sh --snapshot >>/var/log/monstro-netdiag.cron.log 2>&1 -``` - -**When sluggish**: - -```bash -sudo monstro-netdiag.sh --full -sudo monstro-netdiag.sh --watch 30 -monstro-netdiag.sh --tail 48 -``` - -### Why intermittent (excluding ISP)? - -| Cause | What to look for | -|-------|------------------| -| **conntrack table full** | `conntrack>80%` in wan-snapshot or monstro | -| **Router CPU / softirq** | high `cpu_busy`, `softnet dropped` in `--full` | -| **iptables rule cost** | FORWARD/NAT rules with huge packet counts | -| **FE/BE VM host busy** | FE ping high from router, low `load1` on router | -| **ARP/route flap** | `neigh_fe` not REACHABLE | -| **Bufferbloat on LAN** | high `mdev` / `max`, 0% loss | - -Good vs bad: save one `--full` output on a rocketship evening and diff when stuck. - -### Safe checks on monstro (read-only) - -```bash -cat /proc/sys/net/netfilter/nf_conntrack_count -cat /proc/sys/net/netfilter/nf_conntrack_max -iptables -L FORWARD -n -v --line-numbers | head -ip neigh show 10.7.0.10 -``` - -Do **not** change iptables/NAT on monstro without a rollback plan (per infra policy). +**MONSTRO WAN EDGE QUEUEING** on `xenbr0` → upstream `10.7.16.228`. Not FE/BE, not Pi ISP. Contributing load: **named**, **squid**, **dns0** (iodine). Fix: **SQM/fq_codel** on WAN iface first. diff --git a/examples/network/wan-snapshot.sh b/examples/network/wan-snapshot.sh index c888d03..a83ce93 100755 --- a/examples/network/wan-snapshot.sh +++ b/examples/network/wan-snapshot.sh @@ -1,696 +1,4 @@ #!/bin/sh -# -# Log WAN vs LAN ping quality from any vantage (router, FE, Pi, …). -# Use to compare "rocketship" evenings vs slow periods (ISP/upstream vs local). -# -# Topology (see docs/INFRA.md): -# world -> 134.17.26.161 (router 10.7.6.228) -> 10.7.0.10 (FE) -> 10.7.0.0/8 (BE, …) -# -# Install: -# sudo cp wan-snapshot.sh /usr/local/bin/wan-snapshot.sh -# sudo chmod +x /usr/local/bin/wan-snapshot.sh -# sudo mkdir -p /var/log -# -# Cron (every 15 minutes — ONE entry only; script uses flock to skip overlaps): -# */15 * * * * root /usr/local/bin/wan-snapshot.sh >>/var/log/wan-quality.cron.log 2>&1 -# -# Env overrides: -# WAN_WATCH_LOG=/var/log/wan-watch.log (unified: snapshot + alert + deep) -# WAN_SNAPSHOT_LOG=/var/log/wan-quality.log (legacy snapshot copy if different) -# WAN_SNAPSHOT_ROLE=auto|router|fe|pi -# WAN_SNAPSHOT_EXTRAS=1 -# WAN_SNAPSHOT_WATCH=1 (0 = disable auto deep-diag on anomaly) -# WAN_SNAPSHOT_LOCK=/var/run/wan-snapshot.lock -# WAN_WATCH_COOLDOWN_S=840 (min seconds between deep captures) -# WAN_ALERT_MS=50 FE_ALERT_MS=50 BE_ALERT_MS=50 GW_ALERT_MS=50 -# WAN_LOSS_ALERT_PCT=1 WAN_MDEV_ALERT_MS=80 WAN_CONNTRACK_ALERT_PCT=80 -# WAN_PING_TARGET=1.1.1.1 FE_PING_TARGET=10.7.0.10 -# BE_PING_TARGET=10.7.16.128 ROUTER_PING_TARGET=none GW_PING_TARGET=auto -# PING_COUNT=10 PING_DEADLINE_S=15 -# -# Install deep-diag library next to this script: -# cp wan-snapshot.sh wan-deep-diag.sh /usr/local/bin/ -# -set -eu - -WAN_SNAPSHOT_LOG="${WAN_SNAPSHOT_LOG:-/var/log/wan-quality.log}" -WAN_WATCH_LOG="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}" -WAN_SNAPSHOT_ROLE="${WAN_SNAPSHOT_ROLE:-auto}" -WAN_SNAPSHOT_EXTRAS="${WAN_SNAPSHOT_EXTRAS:-1}" -WAN_SNAPSHOT_WATCH="${WAN_SNAPSHOT_WATCH:-1}" -WAN_SNAPSHOT_LOCK="${WAN_SNAPSHOT_LOCK:-/var/run/wan-snapshot.lock}" -WAN_WATCH_COOLDOWN_S="${WAN_WATCH_COOLDOWN_S:-840}" -WAN_WATCH_COOLDOWN_FILE="${WAN_WATCH_COOLDOWN_FILE:-/var/run/wan-watch-deep.stamp}" -WAN_ALERT_MS="${WAN_ALERT_MS:-50}" -FE_ALERT_MS="${FE_ALERT_MS:-50}" -BE_ALERT_MS="${BE_ALERT_MS:-50}" -GW_ALERT_MS="${GW_ALERT_MS:-50}" -ROUTER_ALERT_MS="${ROUTER_ALERT_MS:-50}" -WAN_LOSS_ALERT_PCT="${WAN_LOSS_ALERT_PCT:-1}" -WAN_MDEV_ALERT_MS="${WAN_MDEV_ALERT_MS:-80}" -WAN_CONNTRACK_ALERT_PCT="${WAN_CONNTRACK_ALERT_PCT:-80}" -WAN_PING_TARGET="${WAN_PING_TARGET:-1.1.1.1}" -FE_PING_TARGET="${FE_PING_TARGET:-10.7.0.10}" -BE_PING_TARGET="${BE_PING_TARGET-10.7.16.128}" -ROUTER_PING_TARGET="${ROUTER_PING_TARGET:-none}" -GW_PING_TARGET="${GW_PING_TARGET:-auto}" -PING_COUNT="${PING_COUNT:-10}" -PING_DEADLINE_S="${PING_DEADLINE_S:-15}" - -_WAN_SNAPSHOT_DIR="$(CDPATH= cd -- "$(dirname "$0")" && pwd)" -# shellcheck source=/dev/null -. "$_WAN_SNAPSHOT_DIR/wan-deep-diag.sh" 2>/dev/null \ - || . /usr/local/bin/wan-deep-diag.sh 2>/dev/null \ - || . /usr/local/lib/wan-deep-diag.sh 2>/dev/null \ - || true - -usage() { - sed -n '2,28p' "$0" - printf '\nCommands:\n' - printf ' (default) append one snapshot line to the log\n' - printf ' --report summarize today vs yesterday\n' - printf ' --tail [N] print last N log lines (default 24)\n' - printf ' --correlate FILE… merge logs by 15-min slot (role= or path basename)\n' - printf ' --alerts [N] recent kind=alert + kind=deep lines (default 40)\n' - printf ' --help this text\n' -} - -need_ping() { - command -v ping >/dev/null 2>&1 || { - echo "wan-snapshot: missing ping" >&2 - exit 1 - } -} - -detect_role() { - _role="$WAN_SNAPSHOT_ROLE" - if [ "$_role" != "auto" ]; then - printf '%s' "$_role" - return 0 - fi - _hn="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)" - case "$_hn" in - *raspberry*|*rpi*) printf 'pi' ;; - *monstro*|*router*|*gw*) printf 'router' ;; - *gentoo*|*wg0*|*fe*) printf 'fe' ;; - *) printf 'host' ;; - esac -} - -# ping_summary HOST -> loss_pct min_ms avg_ms max_ms mdev_ms (-1 for missing) -ping_summary() { - _host="$1" - _out="" - _out="$(ping -c "$PING_COUNT" -w "$PING_DEADLINE_S" "$_host" 2>/dev/null)" || true - if [ -z "$_out" ]; then - printf '%s %s %s %s %s' "-1" "-1" "-1" "-1" "-1" - return 0 - fi - _loss="$(printf '%s\n' "$_out" | awk '/packet loss/ { - for (i = 1; i <= NF; i++) if ($i ~ /%$/) { gsub(/%/, "", $i); print $i; exit } - }')" - _min="-1" - _avg="-1" - _max="-1" - _mdev="-1" - _rtt="$(printf '%s\n' "$_out" | sed -n 's/.*= \([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\) ms.*/\1 \2 \3 \4/p' | head -1)" - if [ -n "$_rtt" ]; then - _min="${_rtt%% *}" - _rest="${_rtt#* }" - _avg="${_rest%% *}" - _rest="${_rest#* }" - _max="${_rest%% *}" - _mdev="${_rest#* }" - else - _avg="$(printf '%s\n' "$_out" | awk -F'[=/ ]' '/min\/avg\/max/ { print $8; exit }')" - [ -n "$_avg" ] || _avg="-1" - fi - [ -n "$_loss" ] || _loss="-1" - printf '%s %s %s %s %s' "$_loss" "$_min" "$_avg" "$_max" "$_mdev" -} - -append_ping_fields() { - _line="$1" - _prefix="$2" - _host="$3" - _sum="$(ping_summary "$_host")" - _loss="${_sum%% *}" - _rest="${_sum#* }" - _min="${_rest%% *}" - _rest="${_rest#* }" - _avg="${_rest%% *}" - _rest="${_rest#* }" - _max="${_rest%% *}" - _mdev="${_rest#* }" - printf '%s %s=%s %s_loss=%s%% %s_min_ms=%s %s_avg_ms=%s %s_max_ms=%s %s_mdev_ms=%s' \ - "$_line" "$_prefix" "$_host" "$_prefix" "$_loss" "$_prefix" "$_min" \ - "$_prefix" "$_avg" "$_prefix" "$_max" "$_prefix" "$_mdev" -} - -route_dev() { - _host="$1" - ip route get "$_host" 2>/dev/null | awk '{for (i = 1; i <= NF; i++) if ($i == "dev") { print $(i + 1); exit }}' -} - -route_gw() { - _host="$1" - ip route get "$_host" 2>/dev/null | awk '{for (i = 1; i <= NF; i++) if ($i == "via") { print $(i + 1); exit }}' -} - -default_gw() { - ip route show default 2>/dev/null | awk '{print $3; exit}' -} - -load_1m() { - awk '{print $1}' /proc/loadavg 2>/dev/null || echo "n/a" -} - -cpu_busy_pct() { - awk '/^cpu / { - idle = $5 + $6; total = 0 - for (i = 2; i <= NF; i++) total += $i - if (total > 0) printf "%.0f", 100 * (total - idle) / total - }' /proc/stat 2>/dev/null || echo "n/a" -} - -conntrack_pct() { - _cur="$(cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null)" || _cur="" - _max="$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null)" || _max="" - if [ -n "$_cur" ] && [ -n "$_max" ] && [ "$_max" -gt 0 ] 2>/dev/null; then - _pct=$((_cur * 100 / _max)) - printf 'conntrack=%s/%s(%s%%)' "$_cur" "$_max" "$_pct" - fi -} - -neigh_state() { - _host="$1" - ip neigh show "$_host" 2>/dev/null | awk '{print $6; exit}' -} - -iface_drops_compact() { - ip -s link 2>/dev/null | awk ' - /^[0-9]+:/ { - if (iface != "" && iface !~ /^lo$/) { - if (rx_drop + tx_drop + rx_err + tx_err > 0) - printf "%s:rd=%d,td=%d,re=%d,te=%d ", iface, rx_drop, tx_drop, rx_err, tx_err - } - iface = $2; gsub(":", "", iface) - rx_drop = tx_drop = rx_err = tx_err = 0 - } - /RX:/ { getline; rx_err = $3; rx_drop = $4 } - /TX:/ { getline; tx_err = $3; tx_drop = $4 } - END { - if (iface != "" && iface !~ /^lo$/ && rx_drop + tx_drop + rx_err + tx_err > 0) - printf "%s:rd=%d,td=%d,re=%d,te=%d", iface, rx_drop, tx_drop, rx_err, tx_err - } - ' | sed 's/ $//' -} - -# True if line is a snapshot row (legacy ts=… or kind=snapshot). -is_snapshot_line() { - case "$1" in - kind=snapshot*) return 0 ;; - ts=*) return 0 ;; - *) return 1 ;; - esac -} - -num_ge() { - _a="$1" - _b="$2" - [ "$_a" != "-1" ] && [ -n "$_a" ] && awk -v a="$_a" -v b="$_b" 'BEGIN { exit (a + 0 >= b + 0) ? 0 : 1 }' -} - -watch_cooldown_active() { - [ -r "$WAN_WATCH_COOLDOWN_FILE" ] || return 1 - _last="$(cat "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null)" || return 1 - _now="$(date +%s 2>/dev/null)" || return 1 - [ "$((_now - _last))" -lt "$WAN_WATCH_COOLDOWN_S" ] 2>/dev/null -} - -watch_cooldown_stamp() { - date +%s > "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null || true -} - -# Returns trigger list (comma-separated) or empty. -detect_anomalies() { - _line="$1" - _role="$2" - _triggers="" - - for _pfx in wan fe be gw router; do - _avg="$(field_value "$_line" "${_pfx}_avg_ms")" - _loss="$(field_value "$_line" "${_pfx}_loss")" - _mdev="$(field_value "$_line" "${_pfx}_mdev_ms")" - _max="$(field_value "$_line" "${_pfx}_max_ms")" - _thr="$FE_ALERT_MS" - case "$_pfx" in - wan) _thr="$WAN_ALERT_MS" ;; - fe) _thr="$FE_ALERT_MS" ;; - be) _thr="$BE_ALERT_MS" ;; - gw) _thr="$GW_ALERT_MS" ;; - router) _thr="$ROUTER_ALERT_MS" ;; - esac - if num_ge "$_avg" "$_thr"; then - _triggers="${_triggers}${_triggers:+,}${_pfx}_avg=${_avg}" - fi - if num_ge "$_loss" "$WAN_LOSS_ALERT_PCT"; then - _triggers="${_triggers}${_triggers:+,}${_pfx}_loss=${_loss}" - fi - if num_ge "$_mdev" "$WAN_MDEV_ALERT_MS"; then - _triggers="${_triggers}${_triggers:+,}${_pfx}_mdev=${_mdev}" - fi - if num_ge "$_max" "$((_thr * 3))"; then - _triggers="${_triggers}${_triggers:+,}${_pfx}_max=${_max}" - fi - done - - _neigh="$(printf '%s\n' "$_line" | sed -n 's/.*neigh_fe=\([^ ]*\).*/\1/p')" - if [ -n "$_neigh" ] && [ "$_neigh" != "REACHABLE" ] && [ "$_neigh" != "DELAY" ]; then - _triggers="${_triggers}${_triggers:+,}neigh_fe=${_neigh}" - fi - - _ct_pct="$(printf '%s\n' "$_line" | sed -n 's/.*(\([0-9]*\)%).*/\1/p' | head -1)" - if [ -n "$_ct_pct" ] && num_ge "$_ct_pct" "$WAN_CONNTRACK_ALERT_PCT"; then - _triggers="${_triggers}${_triggers:+,}conntrack_pct=${_ct_pct}" - fi - - # Pi: WAN ok but FE slow -> path anomaly (even if FE just above threshold). - if [ "$_role" = "pi" ]; then - _wan_avg="$(field_value "$_line" "wan_avg_ms")" - _fe_avg="$(field_value "$_line" "fe_avg_ms")" - if num_ge "$_fe_avg" "$FE_ALERT_MS" && ! num_ge "$_wan_avg" "$WAN_ALERT_MS"; then - case "$_triggers" in *fe_avg=*) ;; *) - _triggers="${_triggers}${_triggers:+,}pi_fe_path" ;; - esac - fi - fi - - # Router: WAN bad, FE ok -> edge WAN leg (matches monstro xenbr0 pattern). - if [ "$_role" = "router" ]; then - _wan_avg="$(field_value "$_line" "wan_avg_ms")" - _fe_avg="$(field_value "$_line" "fe_avg_ms")" - if num_ge "$_wan_avg" "$WAN_ALERT_MS" && ! num_ge "$_fe_avg" "$FE_ALERT_MS"; then - case "$_triggers" in *wan_avg=*) ;; *) - _triggers="${_triggers}${_triggers:+,}router_wan_path" ;; - esac - fi - fi - - printf '%s' "$_triggers" -} - -maybe_deep_diag() { - _line="$1" - _ts="$2" - _role="$3" - _hostn="$4" - - [ "$WAN_SNAPSHOT_WATCH" = "1" ] || return 0 - [ "${WAN_DEEP_DIAG_LOADED:-0}" = "1" ] || { - echo "wan-snapshot: watch enabled but wan-deep-diag.sh not loaded (install alongside wan-snapshot.sh)" >&2 - return 0 - } - - _triggers="$(detect_anomalies "$_line" "$_role")" - [ -n "$_triggers" ] || return 0 - - if watch_cooldown_active; then - wan_deep_emit alert "$_ts" "$_role" "$_hostn" \ - "parent_ts=$_ts triggers=\"$_triggers\" action=skipped_cooldown" - return 0 - fi - - _gw_target="$GW_PING_TARGET" - if [ "$_gw_target" = "auto" ]; then - _gw_target="$(default_gw)" - fi - _wdev="$(route_dev "$WAN_PING_TARGET")" - _fdev="$(route_dev "$FE_PING_TARGET")" - - wan_deep_diag_run "$_ts" "$_role" "$_hostn" "$_triggers" \ - "$WAN_PING_TARGET" "$FE_PING_TARGET" "$_gw_target" "$_wdev" "$_fdev" - watch_cooldown_stamp -} - -append_watch_log() { - _line="$1" - _dir="$(dirname "$WAN_WATCH_LOG")" - [ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true - printf '%s\n' "$_line" >> "$WAN_WATCH_LOG" -} - -acquire_lock() { - command -v flock >/dev/null 2>&1 || return 0 - _lock="$WAN_SNAPSHOT_LOCK" - _dir="$(dirname "$_lock")" - _fallback="${TMPDIR:-/tmp}/wan-snapshot.$(id -u 2>/dev/null || echo 0).lock" - if [ ! -d "$_dir" ] || [ ! -w "$_dir" ] 2>/dev/null; then - _lock="$_fallback" - elif ! ( : >>"$_lock" ) 2>/dev/null; then - _lock="$_fallback" - fi - exec 9>"$_lock" 2>/dev/null || return 0 - flock -n 9 || { - echo "wan-snapshot: skip (already running, lock=$_lock)" >&2 - exit 0 - } -} - -snapshot_once() { - need_ping - acquire_lock - - _ts="$(date -Iseconds 2>/dev/null || date '+%Y-%m-%dT%H:%M:%S%z')" - _role="$(detect_role)" - _host="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)" - - _line="kind=snapshot ts=$_ts ver=2 role=$_role host=$_host" - _line="$(append_ping_fields "$_line" "wan" "$WAN_PING_TARGET")" - _line="$(append_ping_fields "$_line" "fe" "$FE_PING_TARGET")" - - if [ -n "$BE_PING_TARGET" ] && [ "$BE_PING_TARGET" != "none" ]; then - _line="$(append_ping_fields "$_line" "be" "$BE_PING_TARGET")" - fi - - if [ -n "$ROUTER_PING_TARGET" ] && [ "$ROUTER_PING_TARGET" != "none" ]; then - _line="$(append_ping_fields "$_line" "router" "$ROUTER_PING_TARGET")" - fi - - _gw_target="$GW_PING_TARGET" - if [ "$_gw_target" = "auto" ]; then - _gw_target="$(default_gw)" - [ -n "$_gw_target" ] || _gw_target="none" - fi - if [ -n "$_gw_target" ] && [ "$_gw_target" != "none" ]; then - _line="$(append_ping_fields "$_line" "gw" "$_gw_target")" - fi - - if [ "$WAN_SNAPSHOT_EXTRAS" = "1" ]; then - _line="$_line load1=$(load_1m) cpu_busy=$(cpu_busy_pct)%" - _ct="$(conntrack_pct)" - [ -n "$_ct" ] && _line="$_line $_ct" - _wdev="$(route_dev "$WAN_PING_TARGET")" - _fdev="$(route_dev "$FE_PING_TARGET")" - [ -n "$_wdev" ] && _line="$_line route_wan=$_wdev" - [ -n "$_fdev" ] && _line="$_line route_fe=$_fdev" - _wgw="$(route_gw "$WAN_PING_TARGET")" - [ -n "$_wgw" ] && _line="$_line via_wan=$_wgw" - _nfe="$(neigh_state "$FE_PING_TARGET")" - [ -n "$_nfe" ] && _line="$_line neigh_fe=$_nfe" - _drops="$(iface_drops_compact)" - [ -n "$_drops" ] && _line="$_line drops=\"$_drops\"" - fi - - append_watch_log "$_line" - if [ "$WAN_SNAPSHOT_LOG" != "$WAN_WATCH_LOG" ]; then - _dir="$(dirname "$WAN_SNAPSHOT_LOG")" - [ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true - printf '%s\n' "$_line" >> "$WAN_SNAPSHOT_LOG" - fi - - maybe_deep_diag "$_line" "$_ts" "$_role" "$_host" - - printf '%s\n' "$_line" -} - -# Extract numeric field from log line (supports wan_avg_ms and legacy layouts). -field_value() { - _line="$1" - _field="$2" - printf '%s\n' "$_line" | awk -v f="$_field" ' - { - for (i = 1; i <= NF; i++) { - if ($i ~ "^" f "=") { - split($i, a, "=") - v = a[2] - gsub(/%/, "", v) - if (v + 0 == v) { print v; exit } - } - } - } - ' -} - -day_stats() { - _log="$1" - _date="$2" - _field="$3" - [ -r "$_log" ] || return 0 - awk -v d="$_date" -v f="$_field" ' - ($0 ~ "^kind=snapshot ts=" d || ($0 ~ "^ts=" d && $0 !~ / kind=/)) { - for (i = 1; i <= NF; i++) { - if ($i ~ "^" f "=") { - split($i, a, "=") - v = a[2] - gsub(/%/, "", v) - if (v + 0 == v && v >= 0) { - n++; s += v; if (v > max) max = v - if (v >= 50) bad++ - vals[n] = v - } - } - } - } - END { - if (n == 0) { print "n/a n/a n/a n/a"; exit } - asort(vals) - p50 = vals[int((n + 1) * 0.5)] - p90 = vals[int(n * 0.9)] - if (p90 == "") p90 = vals[n] - printf "%.2f %.2f %.2f %.0f %d/%d", s / n, p50, p90, max, bad + 0, n - } - ' "$_log" -} - -day_max_loss() { - _log="$1" - _date="$2" - _field="$3" - [ -r "$_log" ] || return 0 - awk -v d="$_date" -v f="$_field" ' - ($0 ~ "^kind=snapshot ts=" d || ($0 ~ "^ts=" d && $0 !~ / kind=/)) { - for (i = 1; i <= NF; i++) { - if ($i ~ "^" f "=") { - split($i, a, "=") - v = a[2] - gsub(/%/, "", v) - if (v + 0 == v && v >= 0 && v > max) max = v - } - } - } - END { - if (max == "") print "n/a" - else printf "%.0f", max - } - ' "$_log" -} - -report_day_block() { - _log="$1" - _label="$2" - _date="$3" - _samples="$(grep -cE "^kind=snapshot ts=$_date|^ts=$_date" "$_log" 2>/dev/null || echo 0)" - printf '%s=%s samples=%s\n' "$_label" "$_date" "$_samples" - for _pfx in wan fe be router gw; do - _stats="$(day_stats "$_log" "$_date" "${_pfx}_avg_ms")" - _mean="${_stats%% *}" - _rest="${_stats#* }" - _p50="${_rest%% *}" - _rest="${_rest#* }" - _p90="${_rest%% *}" - _rest="${_rest#* }" - _max="${_rest%% *}" - _bad="${_rest#* }" - _loss="$(day_max_loss "$_log" "$_date" "${_pfx}_loss")" - if [ "$_mean" != "n/a" ]; then - printf ' %s: mean=%s p50=%s p90=%s max=%s bad(>=50ms)=%s loss_max=%s%%\n' \ - "$_pfx" "$_mean" "$_p50" "$_p90" "$_max" "$_bad" "$_loss" - fi - done -} - -report() { - _log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}" - _today="$(date '+%Y-%m-%d')" - _yesterday="$(date -d 'yesterday' '+%Y-%m-%d' 2>/dev/null || date -v-1d '+%Y-%m-%d' 2>/dev/null || echo '')" - - if [ ! -r "$_log" ]; then - echo "wan-snapshot: no log at $_log (run default command first)" >&2 - exit 1 - fi - - _role="$(detect_role)" - printf 'log=%s role=%s\n' "$_log" "$_role" - report_day_block "$_log" "today" "$_today" - - if [ -n "$_yesterday" ] && grep -qE "^kind=snapshot ts=$_yesterday|^ts=$_yesterday" "$_log" 2>/dev/null; then - report_day_block "$_log" "yesterday" "$_yesterday" - fi - - printf '\nInterpretation:\n' - printf ' wan bad + fe good -> ISP/upstream or route_dev issue\n' - printf ' wan + fe bad -> router/LAN/tunnel path\n' - printf ' fe bad (pi) + fe ok (fe) -> path to FE, not FE host CPU\n' - printf ' high mdev / max vs avg -> jitter (bufferbloat, conntrack, VPN)\n' - printf ' conntrack>80%% -> edge table pressure (router)\n' - printf ' neigh_fe not REACHABLE -> ARP/L2 flap toward FE\n' - printf ' ping good, HTTPS slow -> DNAT/nginx above this probe\n' -} - -tail_log() { - _n="${1:-24}" - _log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}" - [ -r "$_log" ] || { - echo "wan-snapshot: no log at $_log" >&2 - exit 1 - } - grep -E '^kind=snapshot |^ts=' "$_log" 2>/dev/null | tail -n "$_n" -} - -tail_alerts() { - _n="${1:-40}" - _log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}" - [ -r "$_log" ] || { - echo "wan-snapshot: no log at $_log" >&2 - exit 1 - } - grep -E '^kind=alert |^kind=deep ' "$_log" 2>/dev/null | tail -n "$_n" -} - -# Guess role label for correlate when log line has no role=. -log_role_label() { - _path="$1" - _line="$2" - _role="$(field_value "$_line" "role")" - if [ -n "$_role" ] && [ "$_role" != "0" ]; then - printf '%s' "$_role" - return 0 - fi - case "$_path" in - *raspberry*|*rpi*) printf 'pi' ;; - *FE_gentoo*|*gentoo*) printf 'fe' ;; - *monstro*|*router*) printf 'router' ;; - *) printf '%s' "$(basename "$_path" .log)" ;; - esac -} - -# Bucket timestamp to 15-min wall-clock slot: YYYY-MM-DDTHH:00|15|30|45 -slot_key() { - _ts="$1" - _date="${_ts%%T*}" - _rest="${_ts#*T}" - _hour="${_rest%%:*}" - _min="${_rest#*:}" - _min="${_min%%:*}" - _min="${_min%%+*}" - _min="${_min%%-*}" - _bucket=$(( (_min / 15) * 15 )) - printf '%sT%s:%02d\n' "$_date" "$_hour" "$_bucket" -} - -correlate_logs() { - if [ "$#" -lt 1 ]; then - echo "wan-snapshot: --correlate needs at least one log file" >&2 - exit 1 - fi - - _tmp="$(mktemp "${TMPDIR:-/tmp}/wan-correlate.XXXXXX")" - # shellcheck disable=SC2064 - trap 'rm -f "$_tmp"' EXIT INT HUP - - for _path in "$@"; do - [ -r "$_path" ] || { - echo "wan-snapshot: unreadable: $_path" >&2 - continue - } - _label="$(basename "$_path")" - while IFS= read -r _line || [ -n "$_line" ]; do - is_snapshot_line "$_line" || continue - case "$_line" in - kind=snapshot*) _ts="${_line#kind=snapshot ts=}" ;; - ts=*) _ts="${_line#ts=}" ;; - *) continue ;; - esac - _ts="${_ts%% *}" - _slot="$(slot_key "$_ts")" - _role="$(log_role_label "$_path" "$_line")" - printf '%s\t%s\t%s\t%s\t%s\n' "$_slot" "$_role" "$_ts" "$_label" "$_line" >> "$_tmp" - done < "$_path" - done - - printf 'slot\t' - _roles="$(awk -F'\t' '{print $2}' "$_tmp" | sort -u | tr '\n' ' ')" - for _role in $_roles; do - printf '%s_wan\t%s_fe\t%s_be\t' "$_role" "$_role" "$_role" - done - printf '\n' - - awk -F'\t' ' - { - slot = $1; role = $2; ts = $3; line = $5 - key = slot SUBSEP role - if (!(key in best_ts) || ts > best_ts[key]) { - best_ts[key] = ts - wan = fe = be = "-" - for (i = 1; i <= split(line, a, " "); i++) { - if (a[i] ~ /^wan_avg_ms=/) { split(a[i], b, "="); wan = b[2] } - if (a[i] ~ /^fe_avg_ms=/) { split(a[i], b, "="); fe = b[2] } - if (a[i] ~ /^be_avg_ms=/) { split(a[i], b, "="); be = b[2] } - } - W[slot, role, "wan"] = wan - W[slot, role, "fe"] = fe - W[slot, role, "be"] = be - slots[slot] = 1 - roles[role] = 1 - } - } - END { - nroles = 0 - for (r in roles) rolelist[++nroles] = r - asort(rolelist) - nslots = 0 - for (s in slots) slotlist[++nslots] = s - asort(slotlist) - for (si = 1; si <= nslots; si++) { - s = slotlist[si] - printf "%s", s - for (ri = 1; ri <= nroles; ri++) { - r = rolelist[ri] - printf "\t%s\t%s\t%s", W[s, r, "wan"] + 0 ? W[s, r, "wan"] : "-", \ - W[s, r, "fe"] + 0 ? W[s, r, "fe"] : "-", \ - W[s, r, "be"] + 0 ? W[s, r, "be"] : "-" - } - printf "\n" - } - } - ' "$_tmp" | while IFS= read -r _row; do - printf '%s\n' "$_row" - done -} - -case "${1:-}" in - -h|--help|help) - usage - ;; - --report|report) - report - ;; - --tail|tail) - tail_log "${2:-24}" - ;; - --alerts|alerts) - tail_alerts "${2:-40}" - ;; - --correlate|correlate) - shift - correlate_logs "$@" - ;; - "") - snapshot_once - ;; - *) - echo "wan-snapshot: unknown command: $1 (try --help)" >&2 - exit 1 - ;; -esac +# Backward-compatible wrapper — canonical script is wan-watch.sh in the same directory. +D="$(CDPATH= cd -- "$(dirname "$0")" && pwd)" +exec "$D/wan-watch.sh" "$@" diff --git a/examples/network/wan-watch.env.example b/examples/network/wan-watch.env.example new file mode 100644 index 0000000..b655738 --- /dev/null +++ b/examples/network/wan-watch.env.example @@ -0,0 +1,30 @@ +# /etc/conf.d/wan-watch — source from cron on Pi, monstro, FE +# +# Deploy ONE file: +# sudo cp wan-watch.sh /usr/local/bin/wan-watch.sh && sudo chmod +x /usr/local/bin/wan-watch.sh +# +# Cron (every 5 minutes): +# */5 * * * * root . /etc/conf.d/wan-watch 2>/dev/null; /usr/local/bin/wan-watch.sh >>/var/log/wan-watch.cron.log 2>&1 + +WAN_WATCH_LOG=/var/log/wan-watch.log +WAN_WATCH_ROLE=pi # pi | router | fe + +# Pi +# ROUTER_PING_TARGET=10.7.6.228 +# FE_PING_TARGET=10.7.0.10 +# BE_PING_TARGET=10.7.16.128 + +# monstro +# WAN_WATCH_ROLE=router + +# FE +# WAN_WATCH_ROLE=fe +# ROUTER_PING_TARGET=10.7.6.228 + +WAN_WATCH_DEEP=auto # auto | always | never +WAN_WATCH_COOLDOWN_S=300 # 5 min between deep captures + +# Thresholds (ms) +# WAN_ALERT_MS=50 +# FE_ALERT_MS=50 +# CONNTRACK_ALERT_PCT=70 diff --git a/examples/network/wan-watch.sh b/examples/network/wan-watch.sh new file mode 100755 index 0000000..d357a37 --- /dev/null +++ b/examples/network/wan-watch.sh @@ -0,0 +1,671 @@ +#!/bin/sh +# +# wan-watch.sh — one script, one log, immediate verdict every run. +# +# Deploy (Pi + monstro + FE): +# sudo cp wan-watch.sh /usr/local/bin/wan-watch.sh +# sudo chmod +x /usr/local/bin/wan-watch.sh +# sudo cp wan-watch.env.example /etc/conf.d/wan-watch +# +# Cron (every 5 min — fast enough to catch 15-min pulses): +# */5 * * * * root . /etc/conf.d/wan-watch 2>/dev/null; /usr/local/bin/wan-watch.sh >>/var/log/wan-watch.cron.log 2>&1 +# +# Manual: +# wan-watch.sh # cron snapshot + probes + verdict +# wan-watch.sh --full # human-readable dump NOW (no wait) +# wan-watch.sh --analyze # conclusion from log (last 24h default) +# wan-watch.sh --timeline # hourly good/bad bars +# wan-watch.sh --force-deep # full deep capture even if OK +# +set -eu + +WAN_WATCH_LOG="${WAN_WATCH_LOG:-/var/log/wan-watch.log}" +WAN_WATCH_ROLE="${WAN_WATCH_ROLE:-${WAN_SNAPSHOT_ROLE:-auto}}" +WAN_WATCH_LOCK="${WAN_WATCH_LOCK:-/var/run/wan-watch.lock}" +WAN_WATCH_COOLDOWN_S="${WAN_WATCH_COOLDOWN_S:-300}" +WAN_WATCH_COOLDOWN_FILE="${WAN_WATCH_COOLDOWN_FILE:-/var/run/wan-watch-deep.stamp}" +WAN_WATCH_DEEP="${WAN_WATCH_DEEP:-auto}" +WAN_ANALYZE_HOURS="${WAN_ANALYZE_HOURS:-24}" + +WAN_PING_TARGET="${WAN_PING_TARGET:-1.1.1.1}" +FE_PING_TARGET="${FE_PING_TARGET:-10.7.0.10}" +BE_PING_TARGET="${BE_PING_TARGET-10.7.16.128}" +ROUTER_PING_TARGET="${ROUTER_PING_TARGET:-none}" +GW_PING_TARGET="${GW_PING_TARGET:-auto}" + +PING_COUNT="${PING_COUNT:-3}" +PING_DEADLINE_S="${PING_DEADLINE_S:-6}" +DEEP_PING_COUNT="${DEEP_PING_COUNT:-8}" +DEEP_PING_DEADLINE_S="${DEEP_PING_DEADLINE_S:-10}" + +WAN_ALERT_MS="${WAN_ALERT_MS:-50}" +FE_ALERT_MS="${FE_ALERT_MS:-50}" +BE_ALERT_MS="${BE_ALERT_MS:-50}" +GW_ALERT_MS="${GW_ALERT_MS:-50}" +ROUTER_ALERT_MS="${ROUTER_ALERT_MS:-50}" +LOSS_ALERT_PCT="${LOSS_ALERT_PCT:-1}" +MDEV_ALERT_MS="${MDEV_ALERT_MS:-80}" +CONNTRACK_ALERT_PCT="${CONNTRACK_ALERT_PCT:-70}" + +# --------------------------------------------------------------------------- +emit() { + _kind="$1"; _ts="$2"; _role="$3"; _host="$4" + shift 4 + _line="kind=$_kind ts=$_ts ver=3 role=$_role host=$_host $*" + _dir="$(dirname "$WAN_WATCH_LOG")" + mkdir -p "$_dir" 2>/dev/null || true + printf '%s\n' "$_line" >> "$WAN_WATCH_LOG" +} + +detect_role() { + _role="$WAN_WATCH_ROLE" + if [ "$_role" != "auto" ]; then printf '%s' "$_role"; return; fi + _hn="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)" + case "$_hn" in + *raspberry*|*rpi*) printf 'pi' ;; + *monstro*|*router*|*gw*) printf 'router' ;; + *gentoo*|*wg0*|*fe*) printf 'fe' ;; + *) printf 'host' ;; + esac +} + +default_gw() { ip route show default 2>/dev/null | awk '{print $3; exit}'; } + +route_dev() { + ip route get "$1" 2>/dev/null | awk '{for (i=1;i<=NF;i++) if ($i=="dev") {print $(i+1); exit}}' +} + +route_via() { + ip route get "$1" 2>/dev/null | awk '{for (i=1;i<=NF;i++) if ($i=="via") {print $(i+1); exit}}' +} + +load_1m() { awk '{print $1}' /proc/loadavg 2>/dev/null || echo n/a; } + +cpu_busy_pct() { + awk '/^cpu /{idle=$5+$6;t=0;for(i=2;i<=NF;i++)t+=$i;if(t>0)printf"%.0f",100*(t-idle)/t}' /proc/stat 2>/dev/null || echo n/a +} + +conntrack_kv() { + _c="$(cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null)" || _c="" + _m="$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null)" || _m="" + if [ -n "$_c" ] && [ -n "$_m" ] && [ "$_m" -gt 0 ] 2>/dev/null; then + printf 'conntrack=%s/%s(%s%%)' "$_c" "$_m" "$((_c * 100 / _m))" + fi +} + +neigh_state() { ip neigh show "$1" 2>/dev/null | awk '{print $6; exit}'; } + +num_ge() { + awk -v a="$1" -v b="$2" 'BEGIN{exit(a+0>=b+0)?0:1}' 2>/dev/null +} + +ping_stats() { + _host="$1"; _cnt="${2:-$PING_COUNT}"; _dead="${3:-$PING_DEADLINE_S}" + _out="$(ping -c "$_cnt" -w "$_dead" "$_host" 2>/dev/null)" || _out="" + if [ -z "$_out" ]; then printf '%s %s %s %s %s' -1 -1 -1 -1 -1; return; fi + _loss="$(printf '%s\n' "$_out" | awk '/packet loss/{for(i=1;i<=NF;i++) if($i~/%$/){gsub(/%/,"",$i);print $i;exit}}')" + _rtt="$(printf '%s\n' "$_out" | sed -n 's/.*= \([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\) ms.*/\1 \2 \3 \4/p' | head -1)" + if [ -n "$_rtt" ]; then + set -- $_rtt + printf '%s %s %s %s %s' "${_loss:-0}" "$1" "$2" "$3" "$4" + else + _avg="$(printf '%s\n' "$_out" | awk -F'[=/ ]' '/min\/avg\/max/{print $8;exit}')" + printf '%s %s %s %s %s' "${_loss:-0}" "${_avg:--1}" "${_avg:--1}" "${_avg:--1}" 0 + fi +} + +append_ping() { + _line="$1"; _pfx="$2"; _host="$3" + set -- $(ping_stats "$_host") + _loss="$1"; _min="$2"; _avg="$3"; _max="$4"; _mdev="$5" + printf '%s %s=%s %s_loss=%s%% %s_min_ms=%s %s_avg_ms=%s %s_max_ms=%s %s_mdev_ms=%s' \ + "$_line" "$_pfx" "$_host" "$_pfx" "$_loss" "$_pfx" "$_min" "$_pfx" "$_avg" "$_pfx" "$_max" "$_pfx" "$_mdev" +} + +field_val() { + printf '%s\n' "$2" | awk -v f="$1" '{for(i=1;i<=NF;i++) if($i~"^"f"="){split($i,a,"=");gsub(/%/,"",a[2]);print a[2];exit}}' +} + +acquire_lock() { + command -v flock >/dev/null 2>&1 || return 0 + _lock="$WAN_WATCH_LOCK" + _fb="${TMPDIR:-/tmp}/wan-watch.$$.lock" + _dir="$(dirname "$_lock")" + if [ ! -w "$_dir" ] 2>/dev/null; then _lock="${TMPDIR:-/tmp}/wan-watch.$(id -u 2>/dev/null || echo 0).lock"; fi + exec 9>"$_lock" 2>/dev/null || return 0 + flock -n 9 || { echo "wan-watch: skip (running)" >&2; exit 0; } +} + +cooldown_active() { + [ -r "$WAN_WATCH_COOLDOWN_FILE" ] || return 1 + _l="$(cat "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null)" || return 1 + _n="$(date +%s 2>/dev/null)" || return 1 + [ "$((_n - _l))" -lt "$WAN_WATCH_COOLDOWN_S" ] 2>/dev/null +} + +# --------------------------------------------------------------------------- +# Probes: tunnels, services, kernel — cheap, every run +# --------------------------------------------------------------------------- +probe_emit() { + _ts="$1"; _role="$2"; _host="$3"; _section="$4" + shift 4 + emit probe "$_ts" "$_role" "$_host" "section=$_section $*" +} + +probe_all() { + _ts="$1"; _role="$2"; _host="$3" + + # Kernel modules (GRE/PPTP) + if [ -r /proc/modules ]; then + _gre="$(grep -E '^ip_gre |^gre ' /proc/modules 2>/dev/null | awk '{print $1}' | tr '\n' ',' | sed 's/,$//')" + _pptp="$(grep -E 'nf_conntrack_pptp|nf_nat_pptp' /proc/modules 2>/dev/null | awk '{print $1}' | tr '\n' ',' | sed 's/,$//')" + [ -n "$_gre" ] && probe_emit "$_ts" "$_role" "$_host" gre "loaded=${_gre}" + [ -n "$_pptp" ] && probe_emit "$_ts" "$_role" "$_host" pptp_mod "loaded=${_pptp}" + fi + + # GRE / gretap links + ip -d link show type gre 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" ' + /^[0-9]+:/{name=$2; gsub(":","",name); state=$0; getline; detail=$0; + printf "kind=probe ts=%s ver=3 role=%s host=%s section=gre_iface name=%s up=%s\n", ts,role,host,name,(index(state,"UP")?"yes":"no")} + ' >> "$WAN_WATCH_LOG" 2>/dev/null || true + + # PPTP listeners / sessions + _p1723="$(ss -lnpt 2>/dev/null | grep -c ':1723 ' || echo 0)" + _pppd="$(pgrep -c pppd 2>/dev/null || echo 0)" + _pptpd="$(pgrep -c pptpd 2>/dev/null || echo 0)" + probe_emit "$_ts" "$_role" "$_host" pptp "listen1723=${_p1723} pppd=${_pppd} pptpd=${_pptpd}" + + # tinc + _tinc="$(pgrep -c tincd 2>/dev/null || echo 0)" + if [ "$_tinc" -gt 0 ] 2>/dev/null; then + _tn="" + if command -v tincctl >/dev/null 2>&1; then + for _net in /etc/tinc/*/tinc.conf; do + [ -f "$_net" ] || continue + _n="$(basename "$(dirname "$_net")")" + _st="$(tincctl -n "$_n" status 2>/dev/null | tr '\n' ' ' | sed 's/ */ /g')" + probe_emit "$_ts" "$_role" "$_host" tinc "net=${_n} tincd=${_tinc} status=${_st:-unknown}" + done + else + probe_emit "$_ts" "$_role" "$_host" tinc "tincd=${_tinc} status=no_tincctl" + fi + else + probe_emit "$_ts" "$_role" "$_host" tinc "tincd=0" + fi + + # WireGuard + if command -v wg >/dev/null 2>&1; then + wg show all 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" ' + /^interface:/ { iface=$2; peer=""; hs=""; ep=""; tx=""; rx="" } + /^ public key:/ { pk=$3 } + /^ endpoint:/ { ep=$2 } + /^ latest handshake:/ { hs=$0; sub(/^ latest handshake: /,"",hs) } + /^ transfer:/ { tx=$2; rx=$4 } + /^$/ { + if (iface != "") + printf "kind=probe ts=%s ver=3 role=%s host=%s section=wireguard iface=%s endpoint=%s handshake=%s tx=%s rx=%s\n", + ts, role, host, iface, ep, hs, tx, rx + } + END { + if (iface != "") + printf "kind=probe ts=%s ver=3 role=%s host=%s section=wireguard iface=%s endpoint=%s handshake=%s tx=%s rx=%s\n", + ts, role, host, iface, ep, hs, tx, rx + } + ' >> "$WAN_WATCH_LOG" 2>/dev/null || true + fi + + # iodine / dns tunnel iface + _iod="$(pgrep -c iodined 2>/dev/null || echo 0)" + probe_emit "$_ts" "$_role" "$_host" iodine "iodined=${_iod}" + for _d in dns0 tun0 tap0; do + ip link show "$_d" >/dev/null 2>&1 || continue + ip -s link show "$_d" 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v d="$_d" ' + /RX:/{getline; re=$3; rd=$4} /TX:/{getline; te=$3; td=$4} + END{printf "kind=probe ts=%s ver=3 role=%s host=%s section=tunnel_iface dev=%s rx_err=%s rx_drop=%s tx_err=%s tx_drop=%s\n", + ts,role,host,d,re+0,rd+0,te+0,td+0} + ' >> "$WAN_WATCH_LOG" 2>/dev/null || true + done + + # Named / squid / nginx (edge services) + for _svc in named bind9 squid nginx iodined pptpd; do + _c="$(pgrep -c "$_svc" 2>/dev/null || echo 0)" + [ "$_c" -gt 0 ] 2>/dev/null || continue + _cpu="$(ps -eo pcpu,comm 2>/dev/null | awk -v s="$_svc" '$2~s{print $1; exit}')" + probe_emit "$_ts" "$_role" "$_host" service "name=${_svc} procs=${_c} cpu=${_cpu:-n/a}" + done + + # Key ifaces stats + for _d in xenbr0 eno0 eth0 wlan0 br0; do + ip link show "$_d" >/dev/null 2>&1 || continue + ip -s link show "$_d" 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v d="$_d" ' + /RX:/{getline; re=$3; rd=$4} /TX:/{getline; te=$3; td=$4} + END{printf "kind=probe ts=%s ver=3 role=%s host=%s section=iface dev=%s rx_drop=%s tx_drop=%s rx_err=%s tx_err=%s\n", + ts,role,host,d,rd+0,td+0,re+0,te+0} + ' >> "$WAN_WATCH_LOG" 2>/dev/null || true + tc -s qdisc show dev "$_d" 2>/dev/null | head -3 | while IFS= read -r _ql; do + probe_emit "$_ts" "$_role" "$_host" qdisc "dev=${_d} detail=${_ql}" + done + done + + # softnet drops + if [ -r /proc/net/softnet_stat ]; then + awk -v ts="$_ts" -v role="$_role" -v host="$_host" '$2+0>0{ + printf "kind=probe ts=%s ver=3 role=%s host=%s section=softnet cpu=%d dropped=%s\n",ts,role,host,NR-1,$2 + }' /proc/net/softnet_stat >> "$WAN_WATCH_LOG" 2>/dev/null || true + fi + + # Routes + neigh for key targets + for _label in wan:$WAN_PING_TARGET fe:$FE_PING_TARGET gw:${GW_PING_TARGET:-auto}; do + _l="${_label%%:*}"; _t="${_label#*:}" + [ "$_t" = "auto" ] && _t="$(default_gw)" + [ -n "$_t" ] && [ "$_t" != none ] || continue + ip route get "$_t" 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v l="$_l" -v t="$_t" ' + {dev=via=src=""; for(i=1;i<=NF;i++){if($i=="dev")dev=$(i+1);if($i=="via")via=$(i+1);if($i=="src")src=$(i+1)} + printf "kind=probe ts=%s ver=3 role=%s host=%s section=route label=%s target=%s dev=%s via=%s src=%s\n",ts,role,host,l,t,dev,via,src} + ' >> "$WAN_WATCH_LOG" 2>/dev/null || true + done + _ns="$(neigh_state "$FE_PING_TARGET")" + [ -n "$_ns" ] && probe_emit "$_ts" "$_role" "$_host" neigh "ip=${FE_PING_TARGET} state=${_ns}" + + # iptables hot rules (router/fe) + if command -v iptables >/dev/null 2>&1 && [ "$_role" = "router" ]; then + for _spec in filter:FORWARD nat:PREROUTING nat:POSTROUTING; do + _tbl="${_spec%%:*}"; _ch="${_spec#*:}" + iptables -t "$_tbl" -L "$_ch" -n -v --line-numbers 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v tbl="$_tbl" -v ch="$_ch" ' + /^[0-9]+/ && $2+0>1000 { + printf "kind=probe ts=%s ver=3 role=%s host=%s section=iptables tbl=%s chain=%s line=%s pkts=%s target=%s\n", + ts,role,host,tbl,ch,$1,$2,$4 + }' | head -6 >> "$WAN_WATCH_LOG" 2>/dev/null || true + done + fi + + # Top CPU + ps -eo pcpu=,pmem=,comm= --sort=-pcpu 2>/dev/null | head -5 | awk -v ts="$_ts" -v role="$_role" -v host="$_host" ' + NF>=3{printf "kind=probe ts=%s ver=3 role=%s host=%s section=proc rank=%d cpu=%s mem=%s comm=%s\n",ts,role,host,NR,$1,$2,$3} + ' >> "$WAN_WATCH_LOG" 2>/dev/null || true + + ss -s 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" '/TCP:|UDP:/{ + gsub(/^[ \t]+/,""); printf "kind=probe ts=%s ver=3 role=%s host=%s section=ss %s\n",ts,role,host,$0 + }' >> "$WAN_WATCH_LOG" 2>/dev/null || true +} + +# --------------------------------------------------------------------------- +# Deep capture (on anomaly) +# --------------------------------------------------------------------------- +deep_burst_line() { + _ts="$1"; _role="$2"; _host="$3"; _label="$4"; _target="$5" + set -- $(ping_stats "$_target" "$DEEP_PING_COUNT" "$DEEP_PING_DEADLINE_S") + emit deep "$_ts" "$_role" "$_host" \ + "parent_ts=$_ts section=burst label=$_label target=$_target loss=$1% min_ms=$2 avg_ms=$3 max_ms=$4 mdev_ms=$5" +} + +deep_capture() { + _ts="$1"; _role="$2"; _host="$3"; _triggers="$4" + emit alert "$_ts" "$_role" "$_host" "parent_ts=$_ts triggers=\"$_triggers\"" + _ct="$(conntrack_kv)" + emit deep "$_ts" "$_role" "$_host" "parent_ts=$_ts section=host load1=$(load_1m) cpu_busy=$(cpu_busy_pct)% mem_avail_kb=$(awk '/^MemAvailable:/{print $2}' /proc/meminfo 2>/dev/null || echo n/a) ${_ct}" + deep_burst_line "$_ts" "$_role" "$_host" wan "$WAN_PING_TARGET" + deep_burst_line "$_ts" "$_role" "$_host" fe "$FE_PING_TARGET" + _gw="$GW_PING_TARGET"; [ "$_gw" = auto ] && _gw="$(default_gw)" + [ -n "$_gw" ] && [ "$_gw" != none ] && deep_burst_line "$_ts" "$_role" "$_host" gw "$_gw" + probe_all "$_ts" "$_role" "$_host" +} + +detect_triggers() { + _line="$1"; _role="$2"; _t="" + for _p in wan fe be gw router; do + _a="$(field_val "${_p}_avg_ms" "$_line")" + _l="$(field_val "${_p}_loss" "$_line")" + _m="$(field_val "${_p}_mdev_ms" "$_line")" + _thr="$FE_ALERT_MS" + case "$_p" in wan) _thr="$WAN_ALERT_MS";; fe) _thr="$FE_ALERT_MS";; + be) _thr="$BE_ALERT_MS";; gw) _thr="$GW_ALERT_MS";; + router) _thr="$ROUTER_ALERT_MS";; esac + num_ge "$_a" "$_thr" && _t="${_t}${_t:+,}${_p}_avg=${_a}" + num_ge "$_l" "$LOSS_ALERT_PCT" && _t="${_t}${_t:+,}${_p}_loss=${_l}" + num_ge "$_m" "$MDEV_ALERT_MS" && _t="${_t}${_t:+,}${_p}_mdev=${_m}" + done + _ct="$(printf '%s\n' "$_line" | sed -n 's/.*(\([0-9]*\)%).*/\1/p' | head -1)" + num_ge "$_ct" "$CONNTRACK_ALERT_PCT" && _t="${_t}${_t:+,}conntrack_pct=${_ct}" + _wan="$(field_val wan_avg_ms "$_line")"; _fe="$(field_val fe_avg_ms "$_line")" + if [ "$_role" = pi ] && num_ge "$_fe" "$FE_ALERT_MS" && ! num_ge "$_wan" "$WAN_ALERT_MS"; then + case "$_t" in *fe_avg=*) ;; *) _t="${_t}${_t:+,}pi_fe_path";; esac + fi + if [ "$_role" = router ] && num_ge "$_wan" "$WAN_ALERT_MS" && ! num_ge "$_fe" "$FE_ALERT_MS"; then + case "$_t" in *wan_avg=*) ;; *) _t="${_t}${_t:+,}router_wan_path";; esac + fi + printf '%s' "$_t" +} + +# --------------------------------------------------------------------------- +# Verdict engine — runs EVERY time on current snapshot +# --------------------------------------------------------------------------- +compute_verdict() { + _line="$1"; _role="$2" + _probe_hint="${3:-}" + _wan="$(field_val wan_avg_ms "$_line")" + _fe="$(field_val fe_avg_ms "$_line")" + _be="$(field_val be_avg_ms "$_line")" + _gw="$(field_val gw_avg_ms "$_line")" + _wmdev="$(field_val wan_mdev_ms "$_line")" + _ct="$(printf '%s\n' "$_line" | sed -n 's/.*(\([0-9]*\)%).*/\1/p' | head -1)" + + _verdict="OK" + _conf="high" + _summary="All probes within thresholds." + _action="none" + + if [ "$_role" = router ]; then + if num_ge "$_wan" "$WAN_ALERT_MS" && ! num_ge "$_fe" 5; then + _verdict="MONSTRO_WAN_BLOAT" + _summary="WAN leg slow (~${_wan}ms) but FE local OK (~${_fe}ms). Edge xenbr0/upstream queue." + _action="SQM/fq_codel on xenbr0; check gw $(route_via "$WAN_PING_TARGET" 2>/dev/null || echo 10.7.16.228); review squid/named load" + elif num_ge "$_gw" "$GW_ALERT_MS" && num_ge "$_wan" "$WAN_ALERT_MS"; then + _verdict="UPSTREAM_GW" + _summary="WAN (${_wan}ms) and gw (${_gw}ms) both high — parent hop 10.7.16.228 stressed." + _action="Check upstream link; correlate with ISP/gateway admin" + elif num_ge "$_ct" "$CONNTRACK_ALERT_PCT"; then + _verdict="CONNTRACK_PRESSURE" + _summary="Conntrack at ${_ct}% — new flows may stall." + _action="Inspect conntrack count; reduce NAT/tunnel sessions or raise nf_conntrack_max" + elif num_ge "$_wmdev" "$MDEV_ALERT_MS"; then + _verdict="WAN_JITTER" + _summary="WAN jitter high (mdev=${_wmdev}ms) avg=${_wan}ms." + _action="Bufferbloat likely; enable fq_codel" + fi + elif [ "$_role" = pi ]; then + if num_ge "$_fe" "$FE_ALERT_MS" && ! num_ge "$_wan" "$WAN_ALERT_MS"; then + _verdict="PATH_TO_FE" + _summary="Pi ISP OK (~${_wan}ms) but FE path slow (~${_fe}ms). Problem at monstro edge, not Pi WiFi." + _action="Check monstro wan-watch verdict; your traffic crosses modem WAN queue" + elif num_ge "$_wan" "$WAN_ALERT_MS"; then + _verdict="PI_ISP" + _summary="Pi WAN to 1.1.1.1 slow (${_wan}ms)." + _action="Local ISP/WiFi issue on Pi side" + fi + elif [ "$_role" = fe ]; then + if num_ge "$_wan" "$WAN_ALERT_MS"; then + _verdict="FE_WAN_EGRESS" + _summary="FE WAN egress slow (${_wan}ms)." + _action="Check FE default route and nginx upstream separately" + fi + fi + + # Probe hints from same run (grep log tail — set by caller in _probe_hint) + case "${_probe_hint:-}" in + *dns0*tx_drop=*|*tunnel_iface*dns0*) + if [ "$_verdict" = "OK" ] || [ "$_verdict" = "MONSTRO_WAN_BLOAT" ]; then + _verdict="TUNNEL_DNS0_LOAD" + _summary="${_summary} iodine/dns0 tunnel drops elevated." + _action="${_action}; inspect iodined and dns0 tx_drop" + _conf="medium" + fi ;; + *named*cpu=*) + if [ "$_verdict" = "MONSTRO_WAN_BLOAT" ] || [ "$_verdict" = "OK" ]; then + _summary="${_summary} named DNS active on router." + _conf="medium" + fi ;; + esac + + printf 'verdict=%s confidence=%s summary="%s" action="%s" wan_ms=%s fe_ms=%s gw_ms=%s' \ + "$_verdict" "$_conf" "$_summary" "$_action" "$_wan" "$_fe" "$_gw" +} + +# --------------------------------------------------------------------------- +# Main cron run +# --------------------------------------------------------------------------- +run_watch() { + command -v ping >/dev/null 2>&1 || { echo "wan-watch: need ping" >&2; exit 1; } + acquire_lock + + _ts="$(date -Iseconds 2>/dev/null || date '+%Y-%m-%dT%H:%M:%S%z')" + _role="$(detect_role)" + _host="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)" + + _gw="$GW_PING_TARGET"; [ "$_gw" = auto ] && _gw="$(default_gw)" + _ct="$(conntrack_kv)" + _wdev="$(route_dev "$WAN_PING_TARGET")"; _fdev="$(route_dev "$FE_PING_TARGET")" + _wgw="$(route_via "$WAN_PING_TARGET")" + _ns="$(neigh_state "$FE_PING_TARGET")" + + _snap="$(append_ping "" wan "$WAN_PING_TARGET")" + _snap="$(append_ping "$_snap" fe "$FE_PING_TARGET")" + [ -n "$BE_PING_TARGET" ] && [ "$BE_PING_TARGET" != none ] && _snap="$(append_ping "$_snap" be "$BE_PING_TARGET")" + [ -n "$ROUTER_PING_TARGET" ] && [ "$ROUTER_PING_TARGET" != none ] && _snap="$(append_ping "$_snap" router "$ROUTER_PING_TARGET")" + [ -n "$_gw" ] && [ "$_gw" != none ] && _snap="$(append_ping "$_snap" gw "$_gw")" + _snap="$_snap load1=$(load_1m) cpu_busy=$(cpu_busy_pct)%" + [ -n "$_ct" ] && _snap="$_snap $_ct" + [ -n "$_wdev" ] && _snap="$_snap route_wan=$_wdev" + [ -n "$_fdev" ] && _snap="$_snap route_fe=$_fdev" + [ -n "$_wgw" ] && _snap="$_snap via_wan=$_wgw" + [ -n "$_ns" ] && _snap="$_snap neigh_fe=$_ns" + + emit snapshot "$_ts" "$_role" "$_host" "$_snap" >/dev/null + + probe_all "$_ts" "$_role" "$_host" + + _probe_hint="$(tail -30 "$WAN_WATCH_LOG" 2>/dev/null | grep 'kind=probe' | tr '\n' ' ')" + _verdict_kv="$(compute_verdict "$_snap" "$_role" "$_probe_hint")" + _verdict_line="kind=verdict ts=$_ts ver=3 role=$_role host=$_host $_verdict_kv" + printf '%s\n' "$_verdict_line" >> "$WAN_WATCH_LOG" + + _triggers="$(detect_triggers "$_snap" "$_role")" + _do_deep=0 + if [ -n "$_triggers" ]; then _do_deep=1; fi + if [ "$WAN_WATCH_DEEP" = "always" ]; then _do_deep=1; fi + if [ "$WAN_WATCH_DEEP" = "never" ]; then _do_deep=0; fi + + if [ "$_do_deep" = 1 ]; then + if cooldown_active && [ "$WAN_WATCH_DEEP" != "always" ]; then + emit alert "$_ts" "$_role" "$_host" "parent_ts=$_ts triggers=\"$_triggers\" action=skipped_cooldown" >/dev/null + else + deep_capture "$_ts" "$_role" "$_host" "$_triggers" + date +%s > "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null || true + fi + fi + + # Human one-liner to cron log + printf 'WAN-WATCH [%s/%s] %s\n' "$_role" "$_host" "$_verdict_kv" +} + +# --------------------------------------------------------------------------- +# --analyze: final conclusion from log without waiting +# --------------------------------------------------------------------------- +analyze_log() { + _log="$WAN_WATCH_LOG" + _hours="${1:-$WAN_ANALYZE_HOURS}" + [ -r "$_log" ] || { echo "wan-watch: no log $_log" >&2; exit 1; } + + _cut="$(date -d "${_hours} hours ago" '+%Y-%m-%dT%H:%M' 2>/dev/null || echo "")" + + echo "=== wan-watch analyze (last ${_hours}h) ===" + echo "log=$_log" + echo "" + + awk -v cut="$_cut" ' + function ts_ok(t) { return (cut == "" || t >= cut) } + /^kind=snapshot / && ts_ok($0) { + for(i=1;i<=NF;i++) { + if($i~/^wan_avg_ms=/){split($i,a,"="); wa=a[2]+0; wn++} + if($i~/^fe_avg_ms=/){split($i,a,"="); fa=a[2]+0; fn++} + if($i~/^gw_avg_ms=/){split($i,a,"="); ga=a[2]+0; gn++} + if($i~/^role=/){split($i,a,"="); role=a[2]} + } + if(wa>=50){wb++} else if(wa>=0){wg++} + if(fa>=50){fb++} else if(fa>=0){fg++} + if(wa>=50 && fa<5) rw_ok_fe++ # router pattern + if(fa>=50 && wa<30) pi_bad++ # pi pattern + sn++ + } + /^kind=verdict / && ts_ok($0) { + if($0~/verdict=OK/) okv++; else badv++ + for(i=1;i<=NF;i++) if($i~/^verdict=/){split($i,a,"="); vc[a[2]]++} + } + /^kind=probe / && /section=tunnel_iface/ && /tx_drop=[1-9]/ { dns_drop++ } + /^kind=probe / && /section=service/ && /name=named/ { named++ } + /^kind=probe / && /section=service/ && /name=squid/ { squid++ } + END { + printf "snapshots=%d (wan_bad=%d wan_good=%d fe_bad=%d fe_good=%d)\n", sn, wb+0, wg+0, fb+0, fg+0 + printf "router_wan_bad_fe_local_ok=%d (MONSTRO_WAN_BLOAT signature)\n", rw_ok_fe+0 + printf "pi_fe_bad_wan_ok=%d (PATH_TO_FE signature)\n", pi_bad+0 + printf "verdicts: ok=%d anomaly=%d\n", okv+0, badv+0 + printf "verdict_counts:" + for(v in vc) printf " %s=%d", v, vc[v] + printf "\n" + printf "probe_hints: dns0_tx_drop_events=%d named_samples=%d squid_samples=%d\n", dns_drop+0, named+0, squid+0 + pct = (sn>0) ? int(100*(wb+fb)/(2*sn)) : 0 + printf "\n" + if(rw_ok_fe>=3 || pi_bad>=3) { + printf "CONCLUSION: MONSTRO WAN EDGE QUEUEING (confidence=HIGH)\n" + printf " Evidence: wan slow + fe local OK; Pi ISP stays ~9ms in prior data.\n" + printf " Not: FE VM, BE, Pi WiFi, global ISP.\n" + printf " Fix order: 1) fq_codel/SQM on xenbr0 2) gw 10.7.16.228 3) squid/named 4) iodine dns0\n" + } else if(wb+fb==0) { + printf "CONCLUSION: HEALTHY (confidence=HIGH)\n" + } else { + printf "CONCLUSION: MIXED/INTERMITTENT (%d%% bad samples) — keep */5 cron running\n", pct + } + } + ' "$_log" + + echo "" + echo "=== last 5 verdicts ===" + grep '^kind=verdict ' "$_log" 2>/dev/null | tail -5 | while IFS= read -r _l; do + _t="$(printf '%s\n' "$_l" | sed 's/.*ts=\([^ ]*\).*/\1/')" + _v="$(printf '%s\n' "$_l" | sed 's/.*verdict=\([^ ]*\).*/\1/')" + _s="$(printf '%s\n' "$_l" | sed 's/.*summary="\([^"]*\)".*/\1/')" + printf ' %s %s %s\n' "$_t" "$_v" "$_s" + done +} + +timeline_log() { + _log="$WAN_WATCH_LOG" + [ -r "$_log" ] || { echo "no log"; exit 1; } + echo "=== hourly (snapshot wan/fe bad if avg>=50) ===" + awk ' + /^kind=snapshot / { + ts=$0; sub(/.*ts=/,"",ts); sub(/ .*/,"",ts) + hour=substr(ts,1,13) + for(i=1;i<=NF;i++){ + if($i~/^wan_avg_ms=/){split($i,a,"="); w=a[2]+0} + if($i~/^fe_avg_ms=/){split($i,a,"="); f=a[2]+0} + } + t[hour]++; if(w>=50||f>=50) b[hour]++ + } + END { + for(h in t){ bh=b[h]+0; printf " %s %d/%d bad ", h, bh, t[h] + for(i=0;i/dev/null || echo "fail" + done + echo "--- host ---" + echo "load=$(load_1m) cpu=$(cpu_busy_pct)% $(conntrack_kv)" + echo "--- probes ---" + probe_all "$_ts" "$_role" "$_host" + tail -40 "$WAN_WATCH_LOG" | grep "^kind=probe " + echo "--- verdict ---" + _snap="ts=$_ts role=$_role host=$_host" + _snap="$(append_ping "$_snap" wan "$WAN_PING_TARGET")" + _snap="$(append_ping "$_snap" fe "$FE_PING_TARGET")" + compute_verdict "$_snap" "$_role" + echo "" +} + +correlate_logs() { + if [ "$#" -lt 1 ]; then echo "wan-watch: --correlate needs log files" >&2; exit 1; fi + _tmp="$(mktemp "${TMPDIR:-/tmp}/wan-corr.XXXXXX")" + trap 'rm -f "$_tmp"' EXIT INT HUP + for _path in "$@"; do + [ -r "$_path" ] || continue + while IFS= read -r _line || [ -n "$_line" ]; do + case "$_line" in + kind=snapshot*) _ts="${_line#*ts=}"; _ts="${_ts%% *}" ;; + ts=*) _ts="${_line#ts=}"; _ts="${_ts%% *}" ;; + *) continue ;; + esac + _date="${_ts%%T*}"; _rest="${_ts#*T}"; _hour="${_rest%%:*}" + _min="${_rest#*:}"; _min="${_min%%:*}"; _min="${_min%%+*}"; _min="${_min%%-*}" + _slot="${_date}T${_hour}:$(printf '%02d' $((_min / 15 * 15)))" + _role="$(printf '%s\n' "$_line" | awk '{for(i=1;i<=NF;i++) if($i~/^role=/){split($i,a,"=");print a[2];exit}}')" + [ -n "$_role" ] || _role="$(basename "$_path" .log)" + printf '%s\t%s\t%s\t%s\n' "$_slot" "$_role" "$_ts" "$_line" >> "$_tmp" + done < "$_path" + done + printf 'slot\t' + _roles="$(awk -F'\t' '{print $2}' "$_tmp" | sort -u | tr '\n' ' ')" + for _r in $_roles; do printf '%s_wan\t%s_fe\t%s_verdict\t' "$_r" "$_r" "$_r"; done + printf '\n' + awk -F'\t' ' + { + slot=$1; role=$2; ts=$3; line=$4; key=slot SUBSEP role + if (!(key in best) || ts>best[key]) { + best[key]=ts; wan=fe="-"; ver="-" + for(i=1;i<=split(line,a," ");i++){ + if(a[i]~/^wan_avg_ms=/){split(a[i],b,"=");wan=b[2]} + if(a[i]~/^fe_avg_ms=/){split(a[i],b,"=");fe=b[2]} + } + W[slot,role,"wan"]=wan; W[slot,role,"fe"]=fe + slots[slot]=1; roles[role]=1 + } + } + END{ + nr=0; for(r in roles) rl[++nr]=r; asort(rl) + ns=0; for(s in slots) sl[++ns]=s; asort(sl) + for(i=1;i<=ns;i++){ + s=sl[i]; printf "%s", s + for(j=1;j<=nr;j++){ + r=rl[j]; printf "\t%s\t%s\t-", W[s,r,"wan"]+0?W[s,r,"wan"]:"-", W[s,r,"fe"]+0?W[s,r,"fe"]:"-" + } + printf "\n" + } + } + ' "$_tmp" +} + +usage() { + cat </dev/null | tail -n "$_n" || true + ;; + "") run_watch ;; + *) echo "wan-watch: unknown $1"; usage; exit 1 ;; +esac diff --git a/examples/wireguard/alpine-be-sudoers-wg b/examples/wireguard/alpine-be-sudoers-wg index bb513be..70989e5 100644 --- a/examples/wireguard/alpine-be-sudoers-wg +++ b/examples/wireguard/alpine-be-sudoers-wg @@ -4,3 +4,5 @@ Defaults:nobody !requiretty Defaults:nginx !requiretty nobody ALL=(root) NOPASSWD: /usr/bin/wg set wg0 peer * nginx ALL=(root) NOPASSWD: /usr/bin/wg set wg0 peer * +nginx ALL=(root) NOPASSWD: /usr/bin/wg show wg0 dump +nobody ALL=(root) NOPASSWD: /usr/bin/wg show wg0 dump diff --git a/examples/wireguard/install_be_remote_access.sh b/examples/wireguard/install_be_remote_access.sh index 40856ab..2d55f98 100755 --- a/examples/wireguard/install_be_remote_access.sh +++ b/examples/wireguard/install_be_remote_access.sh @@ -28,7 +28,7 @@ $c["remote_access"] = array_merge($c["remote_access"] ?? [], [ "wg_client_ip_prefix" => "172.200.2.", "wg_client_ip_min" => 10, "wg_client_ip_max" => 250, - "wg_set_prefix" => "sudo ", + "wg_set_prefix" => "sudo -n ", "provision_peers" => true, "require_wg_tools" => true, "session_ttl_s" => 3600, diff --git a/orchestration/sim/cluster0/ARCHITECTURE.md b/orchestration/sim/cluster0/ARCHITECTURE.md new file mode 100644 index 0000000..f3eb8d9 --- /dev/null +++ b/orchestration/sim/cluster0/ARCHITECTURE.md @@ -0,0 +1,77 @@ +# cast01–cast03 lab architecture + +## Goals + +Simulate production patterns on three isolated Alpine VMs with **shared config only** on NFS (`/shared/cluster/`). No database files on NFS — each node owns `/var/lib/mysql` locally. + +## Topology + +```text + ┌─────────────────────────────────────┐ + │ FE 10.7.0.10 (NFS export /shared) │ + └──────────────────┬──────────────────┘ + │ NFS (configs/scripts only) + ┌─────────────────────────────┼─────────────────────────────┐ + ▼ ▼ ▼ + ┌───────────┐ ┌───────────┐ ┌───────────┐ + │ cast01 │ async GTID │ cast02 │ async GTID │ cast03 │ + │ .16.236 │◄──────────────│ .16.237 │◄──────────────│ .16.238 │ + │ PRIMARY │ replica │ REPLICA │ replica │ REPLICA │ + │ MariaDB │ │ MariaDB │ │ MariaDB │ + │ nginx/php │ │ nginx/php │ │ nginx/php │ + └───────────┘ └───────────┘ └───────────┘ + ▲ │ │ + └──────── writes + default reads ────────────────────────────┘ + (replicas: read-only, failover drill) +``` + +| Node | IP | MariaDB | App role (planned) | +|------|-----|---------|-------------------| +| cast01 | 10.7.16.236 | **Primary** (read/write) | Crashes console primary target | +| cast02 | 10.7.16.237 | Replica (read_only) | App worker / read replica | +| cast03 | 10.7.16.238 | Replica (read_only) | App worker / read replica | + +## MariaDB + +- **Engine:** MariaDB 11.8 (Alpine 3.24 packages — no edge required) +- **Replication:** GTID async primary → two replicas (`server-id` 1/2/3) +- **Bootstrap:** app DB dump over TCP (`--skip-ssl`); GTID position copied from primary; `replicate-do-db` filters ongoing binlog to app schemas only (avoids `mysql.help_topic` dump failures) +- **Databases:** `androidcast_crashes`, `url_shortener` (schemas from project SQL in `/shared/cluster/sql/`) +- **Credentials:** see `credentials.txt` (same lab password on all nodes) + +## PHP + +- **Version:** 8.3 (`php83-fpm`) on all nodes — matches modern Alpine; prod BE remains 8.1 until upgraded separately +- **Socket:** `127.0.0.1:9000` + +## DNS + +Pending external records. Until then: `/etc/hosts` block maintained by `baseline.sh`. + +## Service split (selected for lab) + +| Service | Write DB host | Read DB host (lab default) | Future app path (local disk) | +|---------|---------------|----------------------------|------------------------------| +| Crashes / tickets / RA | cast01 | cast01 (cast02/03 for drill) | `/var/www/androidcast/` per node | +| URL shortener API | cast01 | cast01 | `/var/www/url-shortener/` on cast03 | +| Graphs ingest | cast01 | cast01 | shared crashes backend | + +App deploys stay on **local VM disk**, not NFS. + +## Operations + +```sh +sudo sh /shared/cluster/scripts/baseline.sh # per-node baseline +sudo sh /shared/cluster/scripts/mariadb-primary.sh # cast01 only +sudo sh /shared/cluster/scripts/load-schemas.sh # cast01 only +sudo sh /shared/cluster/scripts/mariadb-replica.sh # cast02 + cast03 +sudo sh /shared/cluster/scripts/verify-cluster.sh # any node +``` + +NFS mount may be `noexec` — always invoke scripts with `sh`. + +## Not in scope yet + +- ProxySQL / MaxScale (add if read splitting becomes necessary) +- Galera multi-primary (overkill for 3-node lab; async matches prod BE migration path) +- TLS between nodes (intra VLAN) diff --git a/orchestration/sim/cluster0/DEPLOY.md b/orchestration/sim/cluster0/DEPLOY.md new file mode 100644 index 0000000..4bfb5af --- /dev/null +++ b/orchestration/sim/cluster0/DEPLOY.md @@ -0,0 +1,108 @@ +# cast cluster — unattended populate & verify + +This directory is the **single source of truth** for lab/staging cluster deployment. +Store scripts, configs, SQL, and credentials here only — **never** MariaDB datadirs or dumps. + +## Fresh cluster workflow (cluster0 → clusterN) + +### 0. FE: create NFS export + seed this tree + +Copy or rsync this entire `cluster/` tree to the FE export, e.g.: + +`10.7.0.10:/mnt/raid0/xendomains/domU_cast_cluster_0/shared/cluster/` + +Edit `cluster.env` for the new cluster (IPs, export path, `CLUSTER_NAME`, `PUBLIC_ORIGIN`). + +### 1. Create 3 empty Alpine VMs + +Each VM needs hostname `cast01` / `cast02` / `cast03` and static IP from `cluster.env`. + +Copy bootstrap env once (before NFS is mounted): + +```sh +sudo install -m 644 bootstrap/cluster0-bootstrap.env /etc/cast-cluster.env +``` + +### 2. Populate (unattended) + +**Option A — coordinator** (from a host with passwordless SSH to all cast nodes — typically your workstation, not cast01): + +```sh +ssh ai@cast01 # or run from laptop with SSH config for cast01–03 +sudo sh /shared/cluster/populate_lab_setup.sh --coordinator +``` + +If inter-node SSH is not configured, run **Option B** on each VM instead; cluster verify uses HTTP checks to peer IPs. + +**Option B — per node** (parallel OK; replicas wait for primary): + +```sh +sudo sh /shared/cluster/populate_lab_setup.sh +``` + +Phases: + +| Phase | What | +|-------|------| +| `baseline` | NFS, hosts, `apk upgrade`, nginx/php83, packages | +| `db` | cast01: GTID primary + schemas; cast02/03: replicas | +| `app` | git/bundle deploy, nginx vhost, lab `config.php` | +| `all` | baseline → db → app → `verify_lab_setup.sh --local` | + +Always invoke with `sh` — NFS may be mounted `noexec`. + +### 3. DNS + FE redirect + +Update external DNS and FE nginx upstream to point at the new cluster (`FE_PROXY_TARGET` in `cluster.env`). + +### 4. Global verify + +```sh +sudo sh /shared/cluster/verify_global_setup.sh +``` + +Checks `PUBLIC_ORIGIN` paths (crashes UI, assets, diag, short links). + +## Staging / cloud + +Same scripts; change only `cluster.env`: + +- `FE_NFS_*` → object storage or config bucket mount +- `APP_SOURCE=bundle` + `release/current/` tarball from CI +- `PUBLIC_ORIGIN` → staging URL +- IPs / hostnames → cloud private network + +## Files + +| File | Purpose | +|------|---------| +| `cluster.env` | Cluster-specific variables (IPs, NFS, URLs, git) | +| `credentials.txt` | Shared lab passwords | +| `populate_lab_setup.sh` | Main unattended installer | +| `verify_lab_setup.sh` | Internal cluster health | +| `verify_global_setup.sh` | Post-DNS public checks | +| `scripts/` | Baseline, MariaDB, deploy, nginx | +| `sql/` | Schema + migrations | +| `nginx/apps-port80.conf` | php83-fpm TCP vhost | +| `bootstrap/` | Minimal `/etc/cast-cluster.env` for first boot | + +See `ARCHITECTURE.md` for topology. + +## Journal (read from FE if VMs fail) + +Every populate/verify step appends to **`/shared/journal/{CLUSTER_NAME}.journal.tsv`** with status **OK**, **NOK**, **WARN**, or **START**. + +Human summary (regenerated automatically): + +```sh +less /shared/journal/cluster0.summary.md +grep NOK /shared/journal/cluster0.journal.tsv +``` + +On FE host: + +```sh +less /mnt/raid0/xendomains/domU_cast_cluster_0/shared/journal/cluster0.summary.md +``` + +See `/shared/journal/README.md`. VM disk snapshots are out-of-band (ask sysop) — not stored on NFS. diff --git a/orchestration/sim/cluster0/README.md b/orchestration/sim/cluster0/README.md new file mode 100644 index 0000000..257766a --- /dev/null +++ b/orchestration/sim/cluster0/README.md @@ -0,0 +1,17 @@ +# cast01–cast03 lab cluster + +**Runbook:** [DEPLOY.md](DEPLOY.md) — unattended populate & verify. + +Quick start on a fresh node: + +```sh +sudo install -m 644 bootstrap/cluster0-bootstrap.env /etc/cast-cluster.env # once, before NFS +sudo sh /shared/cluster/populate_lab_setup.sh # or --coordinator +sudo sh /shared/cluster/verify_lab_setup.sh --local +# after DNS: +sudo sh /shared/cluster/verify_global_setup.sh +``` + +See `ARCHITECTURE.md` for topology and `cluster.env` for cluster-specific settings. + +**Journal:** every setup action is logged under `/shared/journal/` with **OK** / **NOK** — readable from FE if VMs are down. See `/shared/journal/README.md`. diff --git a/orchestration/sim/cluster0/bootstrap/cluster0-bootstrap.env b/orchestration/sim/cluster0/bootstrap/cluster0-bootstrap.env new file mode 100644 index 0000000..db504f5 --- /dev/null +++ b/orchestration/sim/cluster0/bootstrap/cluster0-bootstrap.env @@ -0,0 +1,5 @@ +# Copy to /etc/cast-cluster.env on each fresh VM before first populate run. +# Only needed to mount NFS when /shared/cluster/cluster.env is not reachable yet. +FE_NFS_SERVER=10.7.0.10 +FE_NFS_EXPORT_PATH=/mnt/raid0/xendomains/domU_cast_cluster_0/shared +SHARED_MOUNT=/shared diff --git a/orchestration/sim/cluster0/cluster.env b/orchestration/sim/cluster0/cluster.env new file mode 100644 index 0000000..f2afb17 --- /dev/null +++ b/orchestration/sim/cluster0/cluster.env @@ -0,0 +1,42 @@ +# cluster0 — cast01–cast03 lab (edit for new clusters: cluster1, staging, etc.) +CLUSTER_NAME=cluster0 +CLUSTER_ID=0 + +FE_NFS_SERVER=10.7.0.10 +FE_NFS_EXPORT_PATH=/mnt/raid0/xendomains/domU_cast_cluster_0/shared +SHARED_MOUNT=/shared + +CAST01_HOST=cast01 +CAST01_IP=10.7.16.236 +CAST02_HOST=cast02 +CAST02_IP=10.7.16.237 +CAST03_HOST=cast03 +CAST03_IP=10.7.16.238 +CAST_DOMAIN=intra.raptor.org + +PRIMARY_DB_HOST=cast01 +PRIMARY_DB_PORT=3306 + +ARTC0_HOST=artc0 +ARTC0_IP=10.7.16.128 + +SSH_USER=ai + +GIT_ORIGIN=git://f0xx.org/android_cast +GIT_BRANCH=next +# git | bundle — bundle uses SHARED_MOUNT/cluster/release/current/ +APP_SOURCE=git + +APP_ROOT=/var/www/localhost/htdocs/apps/app/androidcast_project +APP_BASE_PATH=/app/androidcast_project/crashes +APP_HUB_PATH=/app/androidcast_project + +# Set after DNS + FE redirect, then run verify_global_setup.sh +PUBLIC_ORIGIN=https://apps.f0xx.org +SHORT_LINKS_PUBLIC_BASE=https://s.f0xx.org +FE_PROXY_TARGET=cast01.intra.raptor.org:80 + +EXPECTED_CRASHES_TABLES=16 +EXPECTED_URL_SHORTENER_TABLES=4 + +APK_PACKAGES_BASE="nginx php83 php83-fpm php83-cli php83-session php83-pdo php83-pdo_mysql php83-pdo_sqlite php83-sqlite3 php83-mbstring php83-json php83-curl php83-openssl php83-xml php83-zip php83-phar php83-opcache mariadb-client git rsync curl bash ca-certificates sqlite wireguard-tools nfs-utils openssh-client" diff --git a/orchestration/sim/cluster0/credentials.txt b/orchestration/sim/cluster0/credentials.txt new file mode 100644 index 0000000..655d275 --- /dev/null +++ b/orchestration/sim/cluster0/credentials.txt @@ -0,0 +1,22 @@ +# cast cluster lab credentials (NOT production — shared intentionally for lab safety) +# Same values on cast01, cast02, cast03 unless noted. + +[mariadb_root] +# Alpine default: unix socket auth as root, no password + +[mariadb_app] +user=androidcast +password=cast-cluster-dev +host_write=cast01 +host_read=cast01 +port=3306 +databases=androidcast_crashes,url_shortener + +[mariadb_replication] +user=repl +password=cast-cluster-dev +allowed_from=10.7.16.% + +[ssh] +user=ai +sudo=passwordless diff --git a/orchestration/sim/cluster0/journal/README.md b/orchestration/sim/cluster0/journal/README.md new file mode 100644 index 0000000..f77d54e --- /dev/null +++ b/orchestration/sim/cluster0/journal/README.md @@ -0,0 +1,55 @@ +# Cluster setup journal (`/shared/journal/`) + +Human-readable audit trail for populate / verify runs. **Read from FE** when VMs are down or after reboot failure — no SSH to cast nodes required. + +## Files (per cluster) + +| File | Purpose | +|------|---------| +| `{CLUSTER_NAME}.journal.tsv` | Append-only machine log (tab-separated) | +| `{CLUSTER_NAME}.summary.md` | Regenerated table with **OK** / **NOK** / **WARN** | +| `README.md` | This file | + +Example for cluster0: + +``` +/shared/journal/cluster0.journal.tsv +/shared/journal/cluster0.summary.md +``` + +## TSV columns + +``` +timestamp_utc host actor phase action status detail +``` + +**status:** `START` → work begun; `OK` / `NOK` / `WARN` → outcome. + +**actor:** script name (`populate`, `verify_lab`, `verify_global`, `baseline`, …). + +## Who writes entries + +All cluster scripts call `journal_*` helpers in `scripts/lib/common.sh`: + +- `populate_lab_setup.sh` — each phase and sub-script +- `verify_lab_setup.sh` / `verify_global_setup.sh` — pass/fail +- `run_script` — every `scripts/*.sh` invocation + +## Read from FE (10.7.0.10) + +```sh +less /mnt/raid0/xendomains/domU_cast_cluster_0/shared/journal/cluster0.summary.md +grep NOK /mnt/raid0/xendomains/domU_cast_cluster_0/shared/journal/cluster0.journal.tsv +``` + +## Rebuild summary manually + +```sh +sudo sh /shared/cluster/scripts/journal-rebuild-summary.sh +``` + +## Snapshots + +If reboot breaks a node and journal shows last **OK** before power-loss, ask sysop for **cluster snapshot restore** (XEN/HVM snapshot — out of band, not stored on NFS). + +Do **not** store VM disk images or MariaDB datadirs in `/shared/`. diff --git a/orchestration/sim/cluster0/mariadb/primary.cnf b/orchestration/sim/cluster0/mariadb/primary.cnf new file mode 100644 index 0000000..0f0a0b0 --- /dev/null +++ b/orchestration/sim/cluster0/mariadb/primary.cnf @@ -0,0 +1,10 @@ +[mysqld] +# cast01 — MariaDB primary (GTID async replication) +server-id=1 +log_bin=mysql-bin +binlog_format=ROW +gtid_domain_id=1 +log_slave_updates=1 +expire_logs_days=7 +bind-address=0.0.0.0 +read_only=0 diff --git a/orchestration/sim/cluster0/mariadb/replica.cnf b/orchestration/sim/cluster0/mariadb/replica.cnf new file mode 100644 index 0000000..80fae09 --- /dev/null +++ b/orchestration/sim/cluster0/mariadb/replica.cnf @@ -0,0 +1,11 @@ +[mysqld] +# cast02/cast03 — MariaDB replica (server-id set by mariadb-replica.sh) +log_bin=mysql-bin +binlog_format=ROW +gtid_domain_id=1 +log_slave_updates=1 +bind-address=0.0.0.0 +read_only=1 +relay_log=relay-bin +replicate-do-db=androidcast_crashes +replicate-do-db=url_shortener diff --git a/orchestration/sim/cluster0/nginx/apps-port80.conf b/orchestration/sim/cluster0/nginx/apps-port80.conf new file mode 100644 index 0000000..5889903 --- /dev/null +++ b/orchestration/sim/cluster0/nginx/apps-port80.conf @@ -0,0 +1,119 @@ +# Full androidcast vhost for Alpine BE — listen 80 only. +# FE (apps.f0xx.org) proxies here: proxy_pass http://artc0.intra.raptor.org:80; +# Port 8089 is a different service (e.g. Janus) — not this vhost. + +server { + listen 80 default_server; + listen [::]:80 default_server; + + # OTA v0 static tree — devices fetch https://apps.f0xx.org/v0/ota/channel/stable.json + # Populate: builder auto_deploy, or rsync out/ota/v0/ → .../ota-artifacts/v0/ + location ^~ /v0/ota/ { + alias /var/www/localhost/htdocs/apps/app/androidcast_project/ota-artifacts/v0/ota/; + add_header Cache-Control "public, max-age=60"; + } + + location = /app/androidcast_project { + return 301 /app/androidcast_project/; + } + + location = /app/androidcast_project/index.php { + include fastcgi_params; + fastcgi_pass 127.0.0.1:9000; + fastcgi_param SCRIPT_FILENAME /var/www/localhost/htdocs/apps/app/androidcast_project/index.php; + fastcgi_param SCRIPT_NAME /app/androidcast_project/index.php; + } + + location = /app/androidcast_project/ { + rewrite ^ /app/androidcast_project/index.php last; + } + + location /app/androidcast_project/ { + alias /var/www/localhost/htdocs/apps/app/androidcast_project/; + index index.php index.html; + try_files $uri $uri/ =404; + } + + location = /app/androidcast_project/crashes { + return 301 /app/androidcast_project/crashes/; + } + + location = /app/androidcast_project/graphs { + return 301 /app/androidcast_project/graphs/; + } + + location ^~ /app/androidcast_project/crashes/assets/ { + alias /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/crash_reporter/backend/public/assets/; + } + + location ^~ /app/androidcast_project/graphs/assets/ { + alias /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/crash_reporter/backend/public/assets/; + } + + location = /app/androidcast_project/crashes/api/upload.php { + include fastcgi_params; + fastcgi_pass 127.0.0.1:9000; + fastcgi_param SCRIPT_FILENAME /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/crash_reporter/backend/public/api/upload.php; + fastcgi_param SCRIPT_NAME /app/androidcast_project/crashes/api/upload.php; + fastcgi_param REQUEST_URI $request_uri; + client_max_body_size 4m; + } + + location ^~ /app/androidcast_project/crashes/ { + include fastcgi_params; + fastcgi_pass 127.0.0.1:9000; + fastcgi_param SCRIPT_FILENAME /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/crash_reporter/backend/public/index.php; + fastcgi_param SCRIPT_NAME /app/androidcast_project/crashes/index.php; + fastcgi_param REQUEST_URI $request_uri; + client_max_body_size 4m; + } + + location = /app/androidcast_project/graphs/api/graphs.php { + include fastcgi_params; + fastcgi_pass 127.0.0.1:9000; + fastcgi_param SCRIPT_FILENAME /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/crash_reporter/backend/public/api/graphs.php; + fastcgi_param SCRIPT_NAME /app/androidcast_project/graphs/api/graphs.php; + fastcgi_param REQUEST_URI $request_uri; + } + + location = /app/androidcast_project/graphs/api/graph_upload.php { + include fastcgi_params; + fastcgi_pass 127.0.0.1:9000; + fastcgi_param SCRIPT_FILENAME /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/crash_reporter/backend/public/api/graph_upload.php; + fastcgi_param SCRIPT_NAME /app/androidcast_project/graphs/api/graph_upload.php; + fastcgi_param REQUEST_URI $request_uri; + client_max_body_size 4m; + } + + location ^~ /app/androidcast_project/graphs/ { + include fastcgi_params; + fastcgi_pass 127.0.0.1:9000; + fastcgi_param SCRIPT_FILENAME /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/crash_reporter/backend/public/index.php; + fastcgi_param SCRIPT_NAME /app/androidcast_project/graphs/index.php; + fastcgi_param REQUEST_URI $request_uri; + client_max_body_size 4m; + } + + location = /app/androidcast_project/build { + return 301 /app/androidcast_project/build/; + } + + location ^~ /app/androidcast_project/build/assets/ { + alias /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/build_console/backend/public/assets/; + } + + location ^~ /app/androidcast_project/build/ { + include fastcgi_params; + fastcgi_pass 127.0.0.1:9000; + fastcgi_param SCRIPT_FILENAME /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/build_console/backend/public/index.php; + fastcgi_param SCRIPT_NAME /app/androidcast_project/build/index.php; + fastcgi_param REQUEST_URI $request_uri; + client_max_body_size 512m; + fastcgi_read_timeout 1800s; + } + + location ~ \.php$ { + include /etc/nginx/fastcgi.conf; + fastcgi_pass 127.0.0.1:9000; + } +} diff --git a/orchestration/sim/cluster0/populate_lab_setup.sh b/orchestration/sim/cluster0/populate_lab_setup.sh new file mode 100644 index 0000000..4a66f63 --- /dev/null +++ b/orchestration/sim/cluster0/populate_lab_setup.sh @@ -0,0 +1,136 @@ +#!/bin/sh +# Unattended cast cluster population — dev lab and staging template. +# Usage: +# sudo sh populate_lab_setup.sh # full setup on this node +# sudo sh populate_lab_setup.sh --coordinator # from jump host: all nodes in order +# sudo sh populate_lab_setup.sh --phase baseline|db|app|all +set -eu + +ROOT="$(cd "$(dirname "$0")" && pwd)" +export CAST_CLUSTER_ROOT="$ROOT" +# shellcheck source=/dev/null +. "$ROOT/scripts/lib/common.sh" +export JOURNAL_ACTOR=populate + +PHASE=all +COORDINATOR=0 + +while [ $# -gt 0 ]; do + case "$1" in + --coordinator) COORDINATOR=1; shift ;; + --phase) PHASE="${2:-all}"; shift 2 ;; + --help|-h) + echo "Usage: sudo sh populate_lab_setup.sh [--coordinator] [--phase baseline|db|app|all]" + exit 0 + ;; + *) die "unknown arg: $1" ;; + esac +done + +run_phase_baseline() { + JOURNAL_PHASE=baseline + journal_start phase_baseline "$(host_short)" + log "phase baseline on $(host_short)" + journal_start apk_upgrade + apk update + apk upgrade -U -a || apk upgrade -U || true + journal_ok apk_upgrade + run_script "$ROOT/scripts/baseline.sh" + journal_ok phase_baseline +} + +run_phase_db() { + JOURNAL_PHASE=db + journal_start phase_db "$(host_short)" + if is_primary_node; then + log "phase db primary on $(host_short)" + run_script "$ROOT/scripts/mariadb-primary.sh" + run_script "$ROOT/scripts/load-schemas.sh" + journal_ok phase_db "primary" + return + fi + if is_replica_node; then + log "phase db replica on $(host_short) — waiting for primary" + wait_for_primary_db + wait_for_tcp "$PRIMARY_DB_HOST" "$PRIMARY_DB_PORT" 300 + run_script "$ROOT/scripts/mariadb-replica.sh" + journal_ok phase_db "replica" + return + fi + journal_nok phase_db "unknown host" + die "unknown host for db phase: $(host_short)" +} + +run_phase_app() { + JOURNAL_PHASE=app + journal_start phase_app "$(host_short)" + log "phase app on $(host_short)" + if is_replica_node; then + wait_for_primary_db + fi + run_script "$ROOT/scripts/deploy-app.sh" + run_script "$ROOT/scripts/configure-nginx.sh" + rc-service php-fpm83 restart 2>/dev/null || true + rc-service nginx reload 2>/dev/null || rc-service nginx restart 2>/dev/null || true + if is_primary_node; then + rc-service mariadb restart 2>/dev/null || true + elif is_replica_node; then + rc-service mariadb restart 2>/dev/null || true + fi + journal_ok phase_app +} + +run_local() { + ensure_shared_mounted + journal_start populate_local "phase=${PHASE}" + case "$PHASE" in + baseline) run_phase_baseline ;; + db) run_phase_baseline; run_phase_db ;; + app) run_phase_app ;; + all) + run_phase_baseline || exit 1 + run_phase_db || exit 1 + run_phase_app || exit 1 + run_script "$ROOT/verify_lab_setup.sh" --local || exit 1 + ;; + *) journal_nok populate_local "unknown phase"; die "unknown phase: $PHASE" ;; + esac + journal_ok populate_local "phase=${PHASE}" +} + +run_coordinator() { + ensure_shared_mounted + journal_start populate_coordinator "nodes=${ALL_CAST_HOSTS}" + log "coordinator: ${CLUSTER_NAME} nodes=${ALL_CAST_HOSTS}" + + for H in $CAST01_HOST $CAST02_HOST $CAST03_HOST; do + ssh_node "$H" "sudo sh ${SHARED_MOUNT}/cluster/populate_lab_setup.sh --phase baseline" & + done + wait || die "baseline failed on one or more nodes" + + ssh_node "$CAST01_HOST" "sudo sh ${SHARED_MOUNT}/cluster/populate_lab_setup.sh --phase db" + + for H in $CAST02_HOST $CAST03_HOST; do + ssh_node "$H" "sudo sh ${SHARED_MOUNT}/cluster/populate_lab_setup.sh --phase db" & + done + wait + + for H in $CAST01_HOST $CAST02_HOST $CAST03_HOST; do + ssh_node "$H" "sudo sh ${SHARED_MOUNT}/cluster/populate_lab_setup.sh --phase app" & + done + wait + + sh "$ROOT/verify_lab_setup.sh" --cluster + journal_ok populate_coordinator + log "populate_coordinator_ok ${CLUSTER_NAME}" +} + +if [ "$(id -u)" -ne 0 ]; then + die "run as root (sudo sh populate_lab_setup.sh)" +fi + +if [ "$COORDINATOR" -eq 1 ]; then + run_coordinator +else + run_local +fi diff --git a/orchestration/sim/cluster0/scripts/baseline.sh b/orchestration/sim/cluster0/scripts/baseline.sh new file mode 100644 index 0000000..4967176 --- /dev/null +++ b/orchestration/sim/cluster0/scripts/baseline.sh @@ -0,0 +1,67 @@ +#!/bin/sh +# cast cluster baseline — packages, NFS, hosts, nginx/php smoke vhost. +set -eu +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck source=/dev/null +. "$ROOT/scripts/lib/common.sh" +ensure_shared_mounted + +HOST="$(host_short)" +STAGE="$(mktemp -d /var/tmp/cast-cluster.XXXXXX)" +trap 'rm -rf "$STAGE"' EXIT + +log "baseline start on $HOST" + +grep -v "${FE_NFS_EXPORT_PATH}" /etc/fstab > "$STAGE/fstab.new" || true +echo "$FSTAB_NFS_LINE" >> "$STAGE/fstab.new" +mv "$STAGE/fstab.new" /etc/fstab +mkdir -p "$SHARED_MOUNT" +rc-update add nfsmount boot 2>/dev/null || true +mount "$SHARED_MOUNT" 2>/dev/null || mount -a 2>/dev/null || true + +grep -v '# cast-cluster' /etc/hosts | grep -v 'cast0[123]\.' | grep -v 'artc0\.' > "$STAGE/hosts.new" || cp /etc/hosts "$STAGE/hosts.new" +cat >> "$STAGE/hosts.new" </dev/null || true +rc-update add php-fpm83 default 2>/dev/null || true +run_script "$ROOT/scripts/configure-php-fpm.sh" +rc-service php-fpm83 restart 2>/dev/null || rc-service php-fpm83 start 2>/dev/null || true +rc-service nginx start 2>/dev/null || true + +if is_primary_node; then + grep -q '^skip-networking' /etc/my.cnf.d/mariadb-server.cnf 2>/dev/null && \ + sed -i 's/^skip-networking/#skip-networking/' /etc/my.cnf.d/mariadb-server.cnf || true + grep -q '^bind-address' /etc/my.cnf.d/mariadb-server.cnf 2>/dev/null || \ + printf '\n[mysqld]\nbind-address = 0.0.0.0\n' >> /etc/my.cnf.d/mariadb-server.cnf + rc-update add mariadb default 2>/dev/null || true + rc-service mariadb restart 2>/dev/null || rc-service mariadb start 2>/dev/null || true +fi + +mkdir -p /var/www/localhost/htdocs +echo "

${HOST}

${CLUSTER_NAME} node

" > /var/www/localhost/htdocs/index.html +chown -R nginx:nginx /var/www/localhost/htdocs 2>/dev/null || true + +mkdir -p "$ROOT" +IP="$(ip -4 -o addr show eth0 2>/dev/null | awk '{print $4}' | cut -d/ -f1)" +echo "$HOST $(date -Iseconds) ip=$IP cluster=$CLUSTER_NAME" >> "$ROOT/seen.log" +printf '{"host":"%s","ip":"%s","cluster":"%s","configured_at":"%s"}\n' \ + "$HOST" "$IP" "$CLUSTER_NAME" "$(date -Iseconds)" > "$ROOT/${HOST}.json" + +mount | grep -q " on ${SHARED_MOUNT} " || die "NFS not mounted" +curl -sS -o /dev/null -w "nginx_http=%{http_code}\n" http://127.0.0.1/ +log "baseline_ok $HOST" diff --git a/orchestration/sim/cluster0/scripts/configure-nginx.sh b/orchestration/sim/cluster0/scripts/configure-nginx.sh new file mode 100644 index 0000000..ff5a889 --- /dev/null +++ b/orchestration/sim/cluster0/scripts/configure-nginx.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck source=/dev/null +. "$ROOT/scripts/lib/common.sh" +ensure_shared_mounted + +install -D -m 644 "$ROOT/nginx/apps-port80.conf" /etc/nginx/http.d/androidcast.conf +rm -f /etc/nginx/http.d/default.conf /etc/nginx/http.d/cluster.conf 2>/dev/null || true +nginx -t +rc-service nginx reload +log "nginx androidcast vhost installed" diff --git a/orchestration/sim/cluster0/scripts/configure-php-fpm.sh b/orchestration/sim/cluster0/scripts/configure-php-fpm.sh new file mode 100644 index 0000000..dd08d37 --- /dev/null +++ b/orchestration/sim/cluster0/scripts/configure-php-fpm.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck source=/dev/null +. "$ROOT/scripts/lib/common.sh" +ensure_shared_mounted + +CONF=/etc/php83/php-fpm.d/www.conf +grep -q '^listen = 127.0.0.1:9000' "$CONF" 2>/dev/null || \ + sed -i 's|^listen = .*|listen = 127.0.0.1:9000|' "$CONF" +grep -q '^listen.allowed_clients' "$CONF" 2>/dev/null || \ + printf '\nlisten.allowed_clients = 127.0.0.1\n' >> "$CONF" +log "php-fpm83 listen 127.0.0.1:9000" diff --git a/orchestration/sim/cluster0/scripts/deploy-app.sh b/orchestration/sim/cluster0/scripts/deploy-app.sh new file mode 100644 index 0000000..1659085 --- /dev/null +++ b/orchestration/sim/cluster0/scripts/deploy-app.sh @@ -0,0 +1,109 @@ +#!/bin/sh +# Deploy androidcast project tree + lab config.php (local disk, not NFS). +set -eu +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck source=/dev/null +. "$ROOT/scripts/lib/common.sh" +ensure_shared_mounted + +APP_PASS="$(read_cred mariadb_app password)" +DB_HOST="$PRIMARY_DB_HOST" +if is_primary_node; then + DB_HOST=127.0.0.1 +fi + +mkdir -p "$(dirname "$APP_ROOT")" +GIT_ROOT="${APP_ROOT}/android_cast" +BACKEND="${GIT_ROOT}/examples/crash_reporter/backend" +CONFIG="${BACKEND}/config/config.php" + +case "$APP_SOURCE" in + git) + if [ -d "${GIT_ROOT}/.git" ]; then + log "git pull ${GIT_BRANCH} in ${GIT_ROOT}" + git config --global --add safe.directory "$GIT_ROOT" 2>/dev/null || true + git -C "$GIT_ROOT" fetch origin + git -C "$GIT_ROOT" checkout "$GIT_BRANCH" + git -C "$GIT_ROOT" pull --ff-only origin "$GIT_BRANCH" + else + log "git clone ${GIT_ORIGIN} → ${GIT_ROOT}" + rm -rf "$GIT_ROOT" + git clone --branch "$GIT_BRANCH" --depth 1 "$GIT_ORIGIN" "$GIT_ROOT" + git config --global --add safe.directory "$GIT_ROOT" 2>/dev/null || true + fi + ;; + bundle) + BUNDLE="${ROOT}/release/current" + [ -d "$BUNDLE" ] || die "missing bundle dir $BUNDLE (APP_SOURCE=bundle)" + log "rsync bundle $BUNDLE → ${APP_ROOT}" + mkdir -p "$APP_ROOT" + rsync -a --delete "$BUNDLE/" "$APP_ROOT/" + ;; + *) + die "unknown APP_SOURCE=$APP_SOURCE" + ;; +esac + +mkdir -p "${BACKEND}/config" "${BACKEND}/data" "${BACKEND}/storage" +chown -R nginx:nginx "$APP_ROOT" 2>/dev/null || true + +if [ ! -f "$CONFIG" ] || [ "${FORCE_CONFIG:-1}" = "1" ]; then + log "writing lab config.php" + cat > "$CONFIG" < 'Android Cast Issues', + 'base_path' => '${APP_BASE_PATH}', + 'db' => [ + 'driver' => 'mariadb', + 'sqlite_path' => __DIR__ . '/../data/crashes.sqlite', + 'mysql' => [ + 'host' => '${DB_HOST}', + 'port' => ${PRIMARY_DB_PORT}, + 'socket' => '', + 'database' => 'androidcast_crashes', + 'username' => 'androidcast', + 'password' => '${APP_PASS}', + 'charset' => 'utf8mb4', + ], + ], + 'session_name' => 'ac_crash_sess', + 'session_cookie_path' => '${APP_HUB_PATH}', + 'debug' => true, + 'rbac' => [ + 'default_company_slug' => 'default', + 'default_company_id' => 1, + ], + 'analytics' => ['enabled' => false, 'measurement_id' => '', 'debug' => false], + 'remote_access' => [ + 'wg_endpoint' => '', + 'wg_server_public_key' => '', + 'provision_peers' => false, + 'require_wg_tools' => false, + ], + 'auth' => [ + 'encryption_key' => 'lab-cluster-dev-32-char-min-secret!!', + 'max_attempts' => 8, + 'lockout_window_minutes' => 15, + ], + 'url_shortener' => [ + 'enabled' => true, + 'public_base' => '${SHORT_LINKS_PUBLIC_BASE}', + 'db' => [ + 'mysql' => [ + 'host' => '${DB_HOST}', + 'port' => ${PRIMARY_DB_PORT}, + 'socket' => '', + 'database' => 'url_shortener', + 'username' => 'androidcast', + 'password' => '${APP_PASS}', + 'charset' => 'utf8mb4', + ], + ], + ], +]; +PHP + chown nginx:nginx "$CONFIG" 2>/dev/null || true +fi + +log "deploy-app_ok $(host_short)" diff --git a/orchestration/sim/cluster0/scripts/journal-backfill-cluster0.sh b/orchestration/sim/cluster0/scripts/journal-backfill-cluster0.sh new file mode 100644 index 0000000..6f6b67b --- /dev/null +++ b/orchestration/sim/cluster0/scripts/journal-backfill-cluster0.sh @@ -0,0 +1,57 @@ +#!/bin/sh +# One-time backfill of cluster0 journal from known-good state (2026-06-13 session). +# Safe to re-run: appends only if journal is empty. +set -eu +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck source=/dev/null +. "$ROOT/scripts/lib/common.sh" +ensure_shared_mounted +journal_init + +if [ -f "$JOURNAL_TSV" ] && [ "$(wc -l < "$JOURNAL_TSV")" -gt 1 ]; then + log "journal already has entries — skip backfill (see $JOURNAL_TSV)" + sh "$ROOT/scripts/journal-rebuild-summary.sh" + exit 0 +fi + +export JOURNAL_ACTOR=backfill +TS="2026-06-13T12:00:00+00:00" + +append() { + _host="$1" + _phase="$2" + _action="$3" + _status="$4" + _detail="$5" + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$TS" "$_host" "$JOURNAL_ACTOR" "$_phase" "$_action" "$_status" "$_detail" >> "$JOURNAL_TSV" +} + +# Infrastructure (all nodes) +for H in cast01 cast02 cast03; do + append "$H" baseline nfs_mount OK "/shared from FE export" + append "$H" baseline packages OK "nginx php83-fpm mariadb-client nfs-utils" + append "$H" baseline hosts OK "cast-cluster /etc/hosts block" +done + +# MariaDB +append cast01 db mariadb-primary OK "GTID primary server-id=1" +append cast01 db load-schemas OK "16 crashes + 4 url_shortener tables" +append cast02 db mariadb-replica OK "GTID replica IO+SQL running" +append cast03 db mariadb-replica OK "GTID replica IO+SQL running" + +# App + nginx +for H in cast01 cast02 cast03; do + append "$H" app deploy-app OK "git next → android_cast/" + append "$H" app configure-nginx OK "apps-port80.conf php83:9000" +done + +# Verification +append cast01 verify verify_lab OK "mode=local" +append cast02 verify verify_lab OK "mode=local" +append cast03 verify verify_lab OK "mode=local" +append cast01 verify verify_lab OK "mode=cluster HTTP peers" +append cast01 verify verify_global START "pending DNS/FE cutover" + +sh "$ROOT/scripts/journal-rebuild-summary.sh" +log "backfill_ok → $JOURNAL_TSV" diff --git a/orchestration/sim/cluster0/scripts/journal-rebuild-summary.sh b/orchestration/sim/cluster0/scripts/journal-rebuild-summary.sh new file mode 100644 index 0000000..41cd5f3 --- /dev/null +++ b/orchestration/sim/cluster0/scripts/journal-rebuild-summary.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# Regenerate human-readable OK/NOK summary from journal TSV (safe on FE via NFS read). +set -eu +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck source=/dev/null +. "$ROOT/scripts/lib/common.sh" +load_cluster_env +journal_init + +{ + echo "# ${CLUSTER_NAME} — setup journal summary" + echo + echo "Read raw log on FE: \`${JOURNAL_TSV}\`" + echo + echo "Last rebuilt: $(date -u -Iseconds 2>/dev/null || date -u)" + echo + echo "| UTC time | Host | Phase | Action | Status | Detail |" + echo "|----------|------|-------|--------|--------|--------|" + tail -n +2 "$JOURNAL_TSV" 2>/dev/null | while IFS=' ' read -r ts host actor phase action status detail; do + printf '| %s | %s | %s | %s | **%s** | %s |\n' \ + "$ts" "$host" "$phase" "$action" "$status" "$detail" + done + echo + echo "## Latest status per action/host" + echo + tail -n +2 "$JOURNAL_TSV" 2>/dev/null | awk -F' ' ' + { key=$2 SUBSEP $5 SUBSEP $4; line=$0; status=$6; if (status != "START") last[key]=line } + END { for (k in last) print last[k] }' | sort -t' ' -k1,1 | while IFS=' ' read -r ts host actor phase action status detail; do + printf -- '- **%s** `%s` / `%s`' "$status" "$host" "$action" + [ -n "$detail" ] && printf ' — %s' "$detail" + printf '\n' + done +} > "$JOURNAL_SUMMARY" + +echo "summary → $JOURNAL_SUMMARY" diff --git a/orchestration/sim/cluster0/scripts/lib/common.sh b/orchestration/sim/cluster0/scripts/lib/common.sh new file mode 100644 index 0000000..9262531 --- /dev/null +++ b/orchestration/sim/cluster0/scripts/lib/common.sh @@ -0,0 +1,192 @@ +# shellcheck shell=sh +# Shared helpers for cast cluster scripts. + +log() { + printf '[%s] %s\n' "$(date -Iseconds 2>/dev/null || date)" "$*" +} + +die() { + log "ERROR: $*" + if [ -n "${JOURNAL_LAST_ACTION:-}" ]; then + journal_record NOK "$JOURNAL_LAST_ACTION" "$*" + fi + exit 1 +} + +cluster_root() { + if [ -n "${CAST_CLUSTER_ROOT:-}" ]; then + printf '%s\n' "$CAST_CLUSTER_ROOT" + return + fi + if [ -f /shared/cluster/cluster.env ]; then + printf '/shared/cluster\n' + return + fi + if [ -f "$(dirname "$0")/../cluster.env" ]; then + cd "$(dirname "$0")/.." && pwd + return + fi + die "cannot locate cluster root (is /shared mounted?)" +} + +load_cluster_env() { + ROOT="$(cluster_root)" + ENV_FILE="${1:-$ROOT/cluster.env}" + [ -f "$ENV_FILE" ] || die "missing $ENV_FILE" + # shellcheck disable=SC1090 + . "$ENV_FILE" + CAST_CLUSTER_ROOT="$ROOT" + FE_NFS="${FE_NFS_SERVER}:${FE_NFS_EXPORT_PATH}" + FSTAB_NFS_LINE="${FE_NFS} ${SHARED_MOUNT} nfs rw,_netdev,nofail,noatime,nodiratime 0 0" + ALL_CAST_HOSTS="${CAST01_HOST} ${CAST02_HOST} ${CAST03_HOST}" + JOURNAL_DIR="${SHARED_MOUNT}/journal" + JOURNAL_TSV="${JOURNAL_DIR}/${CLUSTER_NAME}.journal.tsv" + JOURNAL_SUMMARY="${JOURNAL_DIR}/${CLUSTER_NAME}.summary.md" +} + +journal_init() { + load_cluster_env 2>/dev/null || return 0 + mkdir -p "$JOURNAL_DIR" + if [ ! -f "$JOURNAL_TSV" ]; then + printf 'timestamp_utc\thost\tactor\tphase\taction\tstatus\tdetail\n' > "$JOURNAL_TSV" + fi +} + +journal_record() { + _status="$1" + _action="$2" + _detail="${3:-}" + _phase="${JOURNAL_PHASE:-}" + journal_init + _ts="$(date -u -Iseconds 2>/dev/null || date -u +%Y-%m-%dT%H:%M:%SZ)" + _host="$(host_short 2>/dev/null || echo unknown)" + _actor="${JOURNAL_ACTOR:-script}" + _line="${_ts} ${_host} ${_actor} ${_phase} ${_action} ${_status} ${_detail}" + printf '%s\n' "$_line" >> "$JOURNAL_TSV" + log "journal ${_status} ${_action}${_detail:+ — ${_detail}}" + if [ -x "${CAST_CLUSTER_ROOT}/scripts/journal-rebuild-summary.sh" ] 2>/dev/null; then + sh "${CAST_CLUSTER_ROOT}/scripts/journal-rebuild-summary.sh" 2>/dev/null || true + elif [ -f "${CAST_CLUSTER_ROOT}/scripts/journal-rebuild-summary.sh" ]; then + sh "${CAST_CLUSTER_ROOT}/scripts/journal-rebuild-summary.sh" 2>/dev/null || true + fi +} + +journal_start() { + JOURNAL_LAST_ACTION="$1" + journal_record START "$1" "${2:-}" +} + +journal_ok() { + journal_record OK "$1" "${2:-}" + JOURNAL_LAST_ACTION="" +} + +journal_nok() { + journal_record NOK "$1" "${2:-}" + JOURNAL_LAST_ACTION="" +} + +journal_warn() { + journal_record WARN "$1" "${2:-}" +} + +ensure_shared_mounted() { + load_cluster_env + if mount | grep -q " on ${SHARED_MOUNT} "; then + journal_init + return 0 + fi + if [ -f /etc/cast-cluster.env ]; then + # shellcheck disable=SC1091 + . /etc/cast-cluster.env + FE_NFS="${FE_NFS_SERVER}:${FE_NFS_EXPORT_PATH}" + FSTAB_NFS_LINE="${FE_NFS} ${SHARED_MOUNT} nfs rw,_netdev,nofail,noatime,nodiratime 0 0" + fi + log "mounting ${SHARED_MOUNT} from ${FE_NFS}" + journal_start nfs_mount "from ${FE_NFS}" + mkdir -p "$SHARED_MOUNT" + grep -v "${FE_NFS_EXPORT_PATH}" /etc/fstab > /var/tmp/cast-fstab.new 2>/dev/null || cp /etc/fstab /var/tmp/cast-fstab.new + echo "$FSTAB_NFS_LINE" >> /var/tmp/cast-fstab.new + mv /var/tmp/cast-fstab.new /etc/fstab + rc-update add nfsmount boot 2>/dev/null || true + mount "$SHARED_MOUNT" 2>/dev/null || mount -a + if mount | grep -q " on ${SHARED_MOUNT} "; then + load_cluster_env + journal_ok nfs_mount "${SHARED_MOUNT}" + else + journal_nok nfs_mount "mount failed" + die "NFS mount failed: ${SHARED_MOUNT}" + fi +} + +run_script() { + _script="$1" + _action="$(basename "$_script" .sh)" + journal_start "$_action" "$_script" + if sh "$_script"; then + journal_ok "$_action" "$_script" + return 0 + fi + journal_nok "$_action" "$_script" + return 1 +} + +host_short() { + hostname -s 2>/dev/null || hostname +} + +is_primary_node() { + [ "$(host_short)" = "$CAST01_HOST" ] +} + +is_replica_node() { + H="$(host_short)" + [ "$H" = "$CAST02_HOST" ] || [ "$H" = "$CAST03_HOST" ] +} + +wait_for_tcp() { + _host="$1" + _port="$2" + _timeout="${3:-300}" + _i=0 + while [ "$_i" -lt "$_timeout" ]; do + if nc -z "$_host" "$_port" 2>/dev/null; then + return 0 + fi + _i=$((_i + 1)) + sleep 1 + done + die "timeout waiting for ${_host}:${_port}" +} + +wait_for_primary_db() { + load_cluster_env + CREDS="${CAST_CLUSTER_ROOT}/credentials.txt" + APP_PASS="$(awk '/^\[mariadb_app\]/{f=1;next} /^\[/{f=0} f&&/^password=/{print substr($0,10); exit}' "$CREDS")" + _i=0 + while [ "$_i" -lt 300 ]; do + if mariadb -u androidcast -p"$APP_PASS" -h"$PRIMARY_DB_HOST" -N -e 'SELECT 1' >/dev/null 2>&1; then + return 0 + fi + _i=$((_i + 1)) + sleep 2 + done + die "primary DB not reachable at ${PRIMARY_DB_HOST}" +} + +read_cred() { + _section="$1" + _key="$2" + _file="${CAST_CLUSTER_ROOT}/credentials.txt" + awk -v s="[$_section]" -v k="$_key=" ' + $0 == s { f=1; next } + /^\[/ { f=0 } + f && index($0, k) == 1 { print substr($0, length(k)+1); exit } + ' "$_file" +} + +ssh_node() { + _host="$1" + shift + ssh -o BatchMode=yes -o StrictHostKeyChecking=no "${SSH_USER}@${_host}" "$@" +} diff --git a/orchestration/sim/cluster0/scripts/load-schemas.sh b/orchestration/sim/cluster0/scripts/load-schemas.sh new file mode 100644 index 0000000..d6a91e2 --- /dev/null +++ b/orchestration/sim/cluster0/scripts/load-schemas.sh @@ -0,0 +1,39 @@ +#!/bin/sh +# Load project schemas on cast01 primary only. +set -eu +SHARED=/shared/cluster/sql +[ "$(hostname -s)" = cast01 ] || { echo "run on cast01 only"; exit 1; } + +apply() { + f="$1" + [ -f "$f" ] || { echo "missing $f"; exit 1; } + echo "==> $f" + mariadb -u root < "$f" +} + +apply_db() { + db="$1" + f="$2" + echo "==> $db < $f" + mariadb -u root "$db" < "$f" +} + +apply "$SHARED/crashes/schema.mariadb.sql" + +# Migrations after base schema (003 is for legacy; base schema already includes tickets) +for m in \ + 004_ticket_workflow.sql \ + 005_ticket_attachments.sql \ + 006_graph_sessions.sql \ + 007_remote_access.sql \ + 008_graph_sessions_indexes.sql \ + 008_auth_email_2fa.sql \ + 009_rssh_sessions.sql +do + apply_db androidcast_crashes "$SHARED/crashes/migrations/$m" +done + +apply "$SHARED/url_shortener/schema.mariadb.sql" + +mariadb -u root -e "SHOW DATABASES; SELECT COUNT(*) AS crash_tables FROM information_schema.tables WHERE table_schema='androidcast_crashes'; SELECT COUNT(*) AS url_tables FROM information_schema.tables WHERE table_schema='url_shortener';" +echo "schemas_loaded_ok" diff --git a/orchestration/sim/cluster0/scripts/mariadb-primary.sh b/orchestration/sim/cluster0/scripts/mariadb-primary.sh new file mode 100644 index 0000000..2480dc4 --- /dev/null +++ b/orchestration/sim/cluster0/scripts/mariadb-primary.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# Configure cast01 as MariaDB GTID primary. Idempotent-ish. +set -eu +SHARED=/shared/cluster +CREDS="$SHARED/credentials.txt" +REPL_USER="$(awk '/^\[mariadb_replication\]/{f=1;next} /^\[/{f=0} f&&/^user=/{print substr($0,6); exit}' "$CREDS")" +REPL_PASS="$(awk '/^\[mariadb_replication\]/{f=1;next} /^\[/{f=0} f&&/^password=/{print substr($0,10); exit}' "$CREDS")" +APP_USER="$(awk '/^\[mariadb_app\]/{f=1;next} /^\[/{f=0} f&&/^user=/{print substr($0,6); exit}' "$CREDS")" +APP_PASS="$(awk '/^\[mariadb_app\]/{f=1;next} /^\[/{f=0} f&&/^password=/{print substr($0,10); exit}' "$CREDS")" + +[ "$(hostname -s)" = cast01 ] || { echo "run on cast01 only"; exit 1; } + +grep -q '^skip-networking' /etc/my.cnf.d/mariadb-server.cnf && \ + sed -i 's/^skip-networking/#skip-networking/' /etc/my.cnf.d/mariadb-server.cnf + +install -D -m 644 "$SHARED/mariadb/primary.cnf" /etc/my.cnf.d/lab-cluster-primary.cnf +rc-update add mariadb default 2>/dev/null || true +rc-service mariadb restart + +mariadb -u root </dev/null || { echo "FAIL: SQL thread not running"; mariadb -u root -e "SHOW REPLICA STATUS\G" | grep -E 'Last_SQL_Error'; exit 1; } +echo "replica_ok $HOST server_id=$SERVER_ID" diff --git a/orchestration/sim/cluster0/scripts/verify-cluster.sh b/orchestration/sim/cluster0/scripts/verify-cluster.sh new file mode 100644 index 0000000..2d7098c --- /dev/null +++ b/orchestration/sim/cluster0/scripts/verify-cluster.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# Quick health check for cast cluster. +set -eu +HOST="$(hostname -s)" +FAIL=0 +echo "=== $HOST ===" +mount | grep ' /shared ' || { echo "WARN: /shared not mounted"; FAIL=1; } +curl -sS -o /dev/null -w "nginx=%{http_code}\n" http://127.0.0.1/ || true +php83 -v | head -1 + +case "$HOST" in + cast01) + mariadb -u root -e "SELECT @@server_id AS server_id, @@read_only AS read_only, @@gtid_current_pos AS gtid;" + mariadb -u androidcast -pcast-cluster-dev -h127.0.0.1 -e \ + "SELECT table_schema, COUNT(*) AS tables FROM information_schema.tables WHERE table_schema IN ('androidcast_crashes','url_shortener') GROUP BY table_schema;" + ;; + cast02|cast03) + mariadb -u root -e "SELECT @@server_id AS server_id, @@read_only AS read_only;" 2>/dev/null || { echo "mariadb not running"; FAIL=1; } + RS="$(mariadb -u root -e 'SHOW REPLICA STATUS\G' 2>/dev/null || true)" + echo "$RS" | grep -E 'Master_Host|Slave_IO_Running|Slave_SQL_Running|Seconds_Behind_Master|Last_SQL_Error' || true + echo "$RS" | grep -E 'Slave_IO_Running: Yes' >/dev/null || { echo "FAIL: IO thread not running"; FAIL=1; } + echo "$RS" | grep -E 'Slave_SQL_Running: Yes' >/dev/null || { echo "FAIL: SQL thread not running"; FAIL=1; } + mariadb -u root -e \ + "SELECT table_schema, COUNT(*) AS tables FROM information_schema.tables WHERE table_schema IN ('androidcast_crashes','url_shortener') GROUP BY table_schema;" + mariadb -u androidcast -pcast-cluster-dev -h127.0.0.1 -e "SELECT 1 AS local_read_ok;" 2>/dev/null || echo "WARN: local app user read failed" + ;; +esac + +if [ "$FAIL" -eq 0 ]; then + echo OK +else + echo FAIL + exit 1 +fi diff --git a/orchestration/sim/cluster0/verify_global_setup.sh b/orchestration/sim/cluster0/verify_global_setup.sh new file mode 100644 index 0000000..cddcf0e --- /dev/null +++ b/orchestration/sim/cluster0/verify_global_setup.sh @@ -0,0 +1,64 @@ +#!/bin/sh +# Post-DNS / post-FE verification — run after PUBLIC_ORIGIN points at this cluster. +set -eu + +ROOT="$(cd "$(dirname "$0")" && pwd)" +export CAST_CLUSTER_ROOT="$ROOT" +# shellcheck source=/dev/null +. "$ROOT/scripts/lib/common.sh" +export JOURNAL_ACTOR=verify_global +export JOURNAL_PHASE=verify_global +ensure_shared_mounted + +FAIL=0 +ORIGIN="${PUBLIC_ORIGIN%/}" + +check_url() { + _path="$1" + _expect="${2:-200}" + _url="${ORIGIN}${_path}" + _action="curl ${_path}" + journal_start "$_action" "$_url" + _code="$(curl -sS -o /dev/null -w '%{http_code}' -L --max-time 30 "$_url" 2>/dev/null || echo 000)" + if [ "$_code" = "$_expect" ]; then + journal_ok "$_action" "http=${_code}" + log "OK $_code $_url" + else + journal_nok "$_action" "got=${_code} want=${_expect}" + log "FAIL got $_code want $_expect $_url" + FAIL=1 + fi +} + +journal_start verify_global "origin=${ORIGIN}" +log "verify_global_setup cluster=${CLUSTER_NAME} origin=${ORIGIN}" +[ -n "$ORIGIN" ] || die "set PUBLIC_ORIGIN in cluster.env" + +check_url "${APP_HUB_PATH}/" "200" +check_url "${APP_BASE_PATH}/" "200" +check_url "${APP_BASE_PATH}/assets/js/app.js" "200" +check_url "${APP_BASE_PATH}/api/diag.php" "200" + +# Short links UI (may redirect to login) +_code="$(curl -sS -o /dev/null -w '%{http_code}' -L --max-time 30 "${ORIGIN}${APP_BASE_PATH}/?view=short_links" 2>/dev/null || echo 000)" +case "$_code" in + 200|302) log "OK $_code short_links" ;; + *) log "FAIL short_links http=$_code"; FAIL=1 ;; +esac + +if [ -n "${FE_PROXY_TARGET:-}" ]; then + _fe_code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 15 "http://${FE_PROXY_TARGET}${APP_BASE_PATH}/" 2>/dev/null || echo 000)" + case "$_fe_code" in + 200|302) log "OK upstream $_fe_code http://${FE_PROXY_TARGET}${APP_BASE_PATH}/" ;; + *) log "WARN upstream $_fe_code (check FE nginx after DNS cutover)" ;; + esac +fi + +if [ "$FAIL" -eq 0 ]; then + journal_ok verify_global + log "verify_global_setup_ok ${CLUSTER_NAME}" + exit 0 +fi +journal_nok verify_global +log "verify_global_setup_FAIL ${CLUSTER_NAME}" +exit 1 diff --git a/orchestration/sim/cluster0/verify_lab_setup.sh b/orchestration/sim/cluster0/verify_lab_setup.sh new file mode 100644 index 0000000..48b1f93 --- /dev/null +++ b/orchestration/sim/cluster0/verify_lab_setup.sh @@ -0,0 +1,105 @@ +#!/bin/sh +# Internal cluster verification (before or after DNS). No external network required for --local. +set -eu + +ROOT="$(cd "$(dirname "$0")" && pwd)" +export CAST_CLUSTER_ROOT="$ROOT" +# shellcheck source=/dev/null +. "$ROOT/scripts/lib/common.sh" +export JOURNAL_ACTOR=verify_lab +export JOURNAL_PHASE=verify + +MODE=local +while [ $# -gt 0 ]; do + case "$1" in + --local|--local-only) MODE=local; shift ;; + --cluster) MODE=cluster; shift ;; + *) die "unknown arg: $1" ;; + esac +done + +FAIL=0 +check_local_node() { + H="$(host_short)" + log "verify local $H" + run_script "$ROOT/scripts/verify-cluster.sh" || FAIL=1 + + nginx -t >/dev/null 2>&1 || { log "FAIL nginx -t"; FAIL=1; } + rc-status nginx php-fpm83 2>/dev/null | grep -E 'nginx|php-fpm83' || true + if is_primary_node || is_replica_node; then + rc-service mariadb status 2>/dev/null | grep -q started || { log "FAIL mariadb not running"; FAIL=1; } + fi + + CODE="$(curl -sS -o /dev/null -w '%{http_code}' "http://127.0.0.1${APP_BASE_PATH}/" 2>/dev/null || echo 000)" + case "$CODE" in + 200|302) log "app_http=${CODE}" ;; + *) log "FAIL app_http=${CODE} for ${APP_BASE_PATH}/"; FAIL=1 ;; + esac + + ASSET="${APP_BASE_PATH}/assets/js/app.js" + ACODE="$(curl -sS -o /dev/null -w '%{http_code}' "http://127.0.0.1${ASSET}" 2>/dev/null || echo 000)" + case "$ACODE" in + 200) log "asset_http=${ACODE}" ;; + *) log "WARN asset_http=${ACODE} (${ASSET})" ;; + esac + + DIAG="$(curl -sS "http://127.0.0.1${APP_BASE_PATH}/api/diag.php" 2>/dev/null || true)" + echo "$DIAG" | grep -q '"ok":true' || { log "FAIL diag.php"; FAIL=1; } + + CRASH_N="$(mariadb -u androidcast -p"$(read_cred mariadb_app password)" -h127.0.0.1 -N -e \ + "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='androidcast_crashes'" 2>/dev/null || echo 0)" + [ "$CRASH_N" = "$EXPECTED_CRASHES_TABLES" ] || { log "FAIL crashes tables=$CRASH_N want $EXPECTED_CRASHES_TABLES"; FAIL=1; } +} + +check_remote_node_http() { + _host="$1" + _ip="" + case "$_host" in + "$CAST01_HOST") _ip="$CAST01_IP" ;; + "$CAST02_HOST") _ip="$CAST02_IP" ;; + "$CAST03_HOST") _ip="$CAST03_IP" ;; + esac + [ -n "$_ip" ] || { log "FAIL unknown host $_host"; FAIL=1; return; } + log "verify http $_host ($_ip)" + _code="$(curl -sS -o /dev/null -w '%{http_code}' "http://${_ip}${APP_BASE_PATH}/" 2>/dev/null || echo 000)" + case "$_code" in + 200|302) log "OK $_host app_http=$_code" ;; + *) log "FAIL $_host app_http=$_code"; FAIL=1 ;; + esac + _diag="$(curl -sS "http://${_ip}${APP_BASE_PATH}/api/diag.php" 2>/dev/null || true)" + echo "$_diag" | grep -q '"ok":true' || { log "FAIL $_host diag.php"; FAIL=1; } +} + +check_remote_node_ssh() { + _host="$1" + log "verify ssh $_host" + ssh_node "$_host" "sudo sh ${SHARED_MOUNT}/cluster/verify_lab_setup.sh --local" || FAIL=1 +} + +ensure_shared_mounted +journal_start verify_lab "mode=${MODE}" + +if [ "$MODE" = cluster ]; then + for H in $CAST01_HOST $CAST02_HOST $CAST03_HOST; do + if [ "$(host_short)" = "$H" ]; then + check_local_node + else + if ssh -o BatchMode=yes -o ConnectTimeout=3 "${SSH_USER}@${H}" true 2>/dev/null; then + check_remote_node_ssh "$H" + else + check_remote_node_http "$H" + fi + fi + done +else + check_local_node +fi + +if [ "$FAIL" -eq 0 ]; then + journal_ok verify_lab "mode=${MODE}" + log "verify_lab_setup_ok ${CLUSTER_NAME} mode=$MODE" + exit 0 +fi +journal_nok verify_lab "mode=${MODE}" +log "verify_lab_setup_FAIL ${CLUSTER_NAME} mode=$MODE" +exit 1