ac/android_cast
1
0
Fork 0
mirror of git://f0xx.org/android_cast synced 2026-07-29 04:18:09 +03:00
Code Issues Packages Projects Releases Wiki Activity

sync on be, vpn

Browse Source
This commit is contained in:
Anton Afanasyeu
2026-06-13 15:31:18 +02:00
parent f9b73b48da
commit 86055bf509
42 changed files with 2472 additions and 917 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
app/src/main/java/com/foxx/androidcast/dev/DevVpnStatusProbe.java
View File

@@ -57,19 +57,28 @@ public final class DevVpnStatusProbe {
tunnelLevel = DevNetworkSelfTestReport.Level.PASS; tunnelLevel = DevNetworkSelfTestReport.Level.PASS;
} else if (RemoteAccessStatusStore.STATE_FAILURE.equals(tunnelState)) { } else if (RemoteAccessStatusStore.STATE_FAILURE.equals(tunnelState)) {
tunnelLevel = DevNetworkSelfTestReport.Level.FAIL; tunnelLevel = DevNetworkSelfTestReport.Level.FAIL;
} else if (RemoteAccessStatusStore.STATE_WAIT.equals(last.state)
&& isAwaitingWhitelist(last.detail)) {
tunnelLevel = DevNetworkSelfTestReport.Level.PASS;
} else { } else {
tunnelLevel = DevNetworkSelfTestReport.Level.WARN; tunnelLevel = DevNetworkSelfTestReport.Level.WARN;
} }
r.item(tunnelLevel, "tunnel-state", tunnelState); String tunnelLabel = formatTunnelStateLabel(tunnelState, last);
r.item(tunnelLevel, "tunnel-state", tunnelLabel);
if (mode != RemoteAccessMode.DISABLED if (mode != RemoteAccessMode.DISABLED
&& !RemoteAccessStatusStore.STATE_OFF.equals(controlPlane)) { && !RemoteAccessStatusStore.STATE_OFF.equals(controlPlane)) {
DevNetworkSelfTestReport.Level cpLevel = DevNetworkSelfTestReport.Level cpLevel;
RemoteAccessStatusStore.STATE_FAILURE.equals(controlPlane) if (controlPlane.startsWith("ok (")) {
? DevNetworkSelfTestReport.Level.FAIL cpLevel = DevNetworkSelfTestReport.Level.PASS;
: RemoteAccessStatusStore.STATE_SUCCESS.equals(controlPlane) } else if (RemoteAccessStatusStore.STATE_FAILURE.equals(last.state)
? DevNetworkSelfTestReport.Level.PASS && RemoteAccessFailureReason.isControlPlaneMethod(last.method)) {
: DevNetworkSelfTestReport.Level.WARN; cpLevel = DevNetworkSelfTestReport.Level.FAIL;
} else if (RemoteAccessStatusStore.STATE_SUCCESS.equals(last.state)) {
cpLevel = DevNetworkSelfTestReport.Level.PASS;
} else {
cpLevel = DevNetworkSelfTestReport.Level.WARN;
}
r.item(cpLevel, "control-plane", controlPlane); r.item(cpLevel, "control-plane", controlPlane);
} }
@@ -172,6 +181,18 @@ public final class DevVpnStatusProbe {
} }
private static String resolveControlPlaneState(RemoteAccessStatusStore.Snapshot last) { private static String resolveControlPlaneState(RemoteAccessStatusStore.Snapshot last) {
if (RemoteAccessStatusStore.STATE_WAIT.equals(last.state)) {
if (isAwaitingWhitelist(last.detail)) {
return "ok (awaiting whitelist)";
}
if ("no_pending_session".equals(last.detail)) {
return "ok (awaiting operator session)";
}
if ("rate_limited".equals(last.detail)) {
return "ok (poll rate limited)";
}
return last.detail.isEmpty() ? RemoteAccessStatusStore.STATE_WAIT : last.detail;
}
if (!RemoteAccessFailureReason.isControlPlaneMethod(last.method)) { if (!RemoteAccessFailureReason.isControlPlaneMethod(last.method)) {
if (RemoteAccessStatusStore.STATE_IDLE.equals(last.state) if (RemoteAccessStatusStore.STATE_IDLE.equals(last.state)
|| RemoteAccessStatusStore.STATE_SUCCESS.equals(last.state)) { || RemoteAccessStatusStore.STATE_SUCCESS.equals(last.state)) {
@@ -182,6 +203,22 @@ public final class DevVpnStatusProbe {
return last.state; return last.state;
} }
private static boolean isAwaitingWhitelist(String detail) {
return "not_whitelisted".equals(detail);
}
private static String formatTunnelStateLabel(String tunnelState, RemoteAccessStatusStore.Snapshot last) {
if (RemoteAccessStatusStore.STATE_WAIT.equals(last.state)) {
if (isAwaitingWhitelist(last.detail)) {
return "idle (awaiting whitelist — no tunnel expected)";
}
if ("no_pending_session".equals(last.detail)) {
return "idle (awaiting operator open session)";
}
}
return tunnelState;
}
private static String resolveDisplayedFailReason( private static String resolveDisplayedFailReason(
Context context, Context context,
RemoteAccessMode mode, RemoteAccessMode mode,
@@ -191,10 +228,12 @@ public final class DevVpnStatusProbe {
if (mode == RemoteAccessMode.DISABLED) { if (mode == RemoteAccessMode.DISABLED) {
return ""; return "";
} }
if (!RemoteAccessStatusStore.STATE_FAILURE.equals(last.state)) {
return "";
}
String expanded = RemoteAccessFailureReason.expandForDisplay(context, last.method, last.failReason); String expanded = RemoteAccessFailureReason.expandForDisplay(context, last.method, last.failReason);
if (RemoteAccessFailureReason.isControlPlaneMethod(last.method) if (RemoteAccessFailureReason.isControlPlaneMethod(last.method)) {
&& RemoteAccessStatusStore.STATE_FAILURE.equals(last.state)) {
if (!RemoteAccessStatusStore.STATE_SUCCESS.equals(tunnelState)) { if (!RemoteAccessStatusStore.STATE_SUCCESS.equals(tunnelState)) {
return "[heartbeat] " + expanded; return "[heartbeat] " + expanded;
} }
@@ -204,8 +243,7 @@ public final class DevVpnStatusProbe {
&& RemoteAccessStatusStore.STATE_FAILURE.equals(tunnelState)) { && RemoteAccessStatusStore.STATE_FAILURE.equals(tunnelState)) {
return "[tunnel] " + expanded; return "[tunnel] " + expanded;
} }
if (RemoteAccessStatusStore.STATE_FAILURE.equals(controlPlane) if (!expanded.isEmpty()) {
&& !expanded.isEmpty()) {
return "[heartbeat] " + expanded; return "[heartbeat] " + expanded;
} }
return ""; return "";

2
app/src/main/java/com/foxx/androidcast/remoteaccess/RemoteAccessFailureReason.java
View File

@@ -14,7 +14,7 @@ public final class RemoteAccessFailureReason {
public static String fromPollException(Context context, Exception e) { public static String fromPollException(Context context, Exception e) {
String host = heartbeatHost(context); String host = heartbeatHost(context);
if (e instanceof SocketTimeoutException) { if (e instanceof SocketTimeoutException) {
return "heartbeat HTTP timeout — device→" + host + " (15s connect/read)"; return "heartbeat HTTP timeout — device→" + host + " (30s connect/read)";
} }
if (e instanceof UnknownHostException) { if (e instanceof UnknownHostException) {
return "heartbeat DNS failed — " + host + " (" + safeMsg(e) + ")"; return "heartbeat DNS failed — " + host + " (" + safeMsg(e) + ")";

2
app/src/main/java/com/foxx/androidcast/remoteaccess/RemoteAccessHeartbeatClient.java
View File

@@ -18,7 +18,7 @@ import java.nio.charset.StandardCharsets;
/** POST heartbeat type:ra to BE (shared by RemoteAccessService). */ /** POST heartbeat type:ra to BE (shared by RemoteAccessService). */
public final class RemoteAccessHeartbeatClient { public final class RemoteAccessHeartbeatClient {
private static final String TAG = "RemoteAccessHB"; private static final String TAG = "RemoteAccessHB";
private static final int HTTP_TIMEOUT_MS = 15_000; private static final int HTTP_TIMEOUT_MS = 30_000;
private RemoteAccessHeartbeatClient() {} private RemoteAccessHeartbeatClient() {}

7
app/src/main/java/com/foxx/androidcast/remoteaccess/RemoteAccessService.java
View File

@@ -170,11 +170,12 @@ public final class RemoteAccessService extends Service {
RemoteAccessStatusStore.INITIATOR_BE, RemoteAccessStatusStore.INITIATOR_BE,
raAction, resp.optString("session_id", "")); raAction, resp.optString("session_id", ""));
} else { } else {
String waitReason = resp.optString("reason", "");
RemoteAccessStatusStore.record( RemoteAccessStatusStore.record(
this, mode, RemoteAccessStatusStore.STATE_IDLE, this, mode, RemoteAccessStatusStore.STATE_WAIT,
"RemoteAccessService.runPoll", "RemoteAccessService.runPoll",
RemoteAccessStatusStore.INITIATOR_DEVICE, RemoteAccessStatusStore.INITIATOR_BE,
"", resp.optString("session_id", "")); "", waitReason, resp.optString("session_id", ""));
} }
} catch (Exception e) { } catch (Exception e) {
Log.w(TAG, "poll failed: " + e.getMessage()); Log.w(TAG, "poll failed: " + e.getMessage());

20
app/src/main/java/com/foxx/androidcast/remoteaccess/RemoteAccessStatusStore.java
View File

@@ -20,12 +20,15 @@ public final class RemoteAccessStatusStore {
public static final String STATE_FAILURE = "failure"; public static final String STATE_FAILURE = "failure";
public static final String STATE_CONNECTING = "connecting"; public static final String STATE_CONNECTING = "connecting";
public static final String STATE_IDLE = "idle"; public static final String STATE_IDLE = "idle";
public static final String STATE_WAIT = "wait";
public static final String STATE_OFF = "off"; public static final String STATE_OFF = "off";
public static final String INITIATOR_DEVICE = "device"; public static final String INITIATOR_DEVICE = "device";
public static final String INITIATOR_BE = "backend"; public static final String INITIATOR_BE = "backend";
public static final String INITIATOR_USER = "user"; public static final String INITIATOR_USER = "user";
private static final String KEY_DETAIL = "detail";
private RemoteAccessStatusStore() {} private RemoteAccessStatusStore() {}
public static void record( public static void record(
@@ -36,6 +39,18 @@ public final class RemoteAccessStatusStore {
String initiator, String initiator,
String failReason, String failReason,
String sessionId) { String sessionId) {
record(context, mode, state, method, initiator, failReason, "", sessionId);
}
public static void record(
Context context,
RemoteAccessMode mode,
String state,
String method,
String initiator,
String failReason,
String detail,
String sessionId) {
String type = mode == null || mode == RemoteAccessMode.DISABLED String type = mode == null || mode == RemoteAccessMode.DISABLED
? "off" : mode.name().toLowerCase(); ? "off" : mode.name().toLowerCase();
prefs(context).edit() prefs(context).edit()
@@ -44,6 +59,7 @@ public final class RemoteAccessStatusStore {
.putString(KEY_METHOD, method != null ? method : "") .putString(KEY_METHOD, method != null ? method : "")
.putString(KEY_INITIATOR, initiator != null ? initiator : "") .putString(KEY_INITIATOR, initiator != null ? initiator : "")
.putString(KEY_FAIL_REASON, failReason != null ? failReason : "") .putString(KEY_FAIL_REASON, failReason != null ? failReason : "")
.putString(KEY_DETAIL, detail != null ? detail : "")
.putString(KEY_SESSION_ID, sessionId != null ? sessionId : "") .putString(KEY_SESSION_ID, sessionId != null ? sessionId : "")
.putLong(KEY_UPDATED_MS, System.currentTimeMillis()) .putLong(KEY_UPDATED_MS, System.currentTimeMillis())
.apply(); .apply();
@@ -57,6 +73,7 @@ public final class RemoteAccessStatusStore {
p.getString(KEY_METHOD, ""), p.getString(KEY_METHOD, ""),
p.getString(KEY_INITIATOR, ""), p.getString(KEY_INITIATOR, ""),
p.getString(KEY_FAIL_REASON, ""), p.getString(KEY_FAIL_REASON, ""),
p.getString(KEY_DETAIL, ""),
p.getString(KEY_SESSION_ID, ""), p.getString(KEY_SESSION_ID, ""),
p.getLong(KEY_UPDATED_MS, 0L)); p.getLong(KEY_UPDATED_MS, 0L));
} }
@@ -71,6 +88,7 @@ public final class RemoteAccessStatusStore {
public final String method; public final String method;
public final String initiator; public final String initiator;
public final String failReason; public final String failReason;
public final String detail;
public final String sessionId; public final String sessionId;
public final long updatedMs; public final long updatedMs;
@@ -80,6 +98,7 @@ public final class RemoteAccessStatusStore {
String method, String method,
String initiator, String initiator,
String failReason, String failReason,
String detail,
String sessionId, String sessionId,
long updatedMs) { long updatedMs) {
this.type = type != null ? type : "off"; this.type = type != null ? type : "off";
@@ -87,6 +106,7 @@ public final class RemoteAccessStatusStore {
this.method = method != null ? method : ""; this.method = method != null ? method : "";
this.initiator = initiator != null ? initiator : ""; this.initiator = initiator != null ? initiator : "";
this.failReason = failReason != null ? failReason : ""; this.failReason = failReason != null ? failReason : "";
this.detail = detail != null ? detail : "";
this.sessionId = sessionId != null ? sessionId : ""; this.sessionId = sessionId != null ? sessionId : "";
this.updatedMs = updatedMs; this.updatedMs = updatedMs;
} }

28
docs/VPN_DEMO_GAPS_20260613.md Normal file
View File

@@ -0,0 +1,28 @@
# VPN / Remote Access demo — gap report (2026-06-13)
See also [REMOTE_ACCESS_VALIDATION.md](../REMOTE_ACCESS_VALIDATION.md), [REMOTE_ACCESS_IMPL.md](../REMOTE_ACCESS_IMPL.md).
## Fixed today (prod BE)
- `config.php` **`remote_access` block** was missing — blocked real WireGuard connect credentials
- Installed via `examples/wireguard/install_be_remote_access.sh`
- **VPN traffic counters** in admin expanded device row (`WireGuardTrafficStats.php`)
- sudoers: `wg show wg0 dump` for nginx FPM user
## Verified OK
- `wg0` on BE, endpoint `ra.apps.f0xx.org:45340`
- UDP 45340 open on public IP
- Heartbeat disable/enable API
- Mobile boot restores RA mode (`RemoteAccessCoordinator.onBootIfNeeded`)
## Open (PO / device)
1. Full `ra_e2e_cli.sh` — needs admin login on laptop
2. UDP handshake + ping — needs phone or `ra_udp_debug.sh --apply`
3. SSH passthrough — RSSH track, not WG v1
4. Doc port references still say 51820 in places; prod uses **45340**
5. Stale wg peer with `allowed ips: (none)` — manual cleanup
6. Persistent traffic history — live stats only today
Journal: `/shared/journal/cluster0.journal.tsv` (tag `vpn-demo`).

28
examples/crash_reporter/backend/public/assets/js/remote_access.js
View File

@@ -81,6 +81,14 @@
return d.innerHTML; return d.innerHTML;
} }
function formatBytes(n) {
const b = Number(n) || 0;
if (b < 1024) return b + ' B';
if (b < 1024 * 1024) return (b / 1024).toFixed(1) + ' KiB';
if (b < 1024 * 1024 * 1024) return (b / (1024 * 1024)).toFixed(2) + ' MiB';
return (b / (1024 * 1024 * 1024)).toFixed(2) + ' GiB';
}
function getColWidths() { function getColWidths() {
try { try {
const w = JSON.parse(lsGet(STORAGE.colWidths, '{}')); const w = JSON.parse(lsGet(STORAGE.colWidths, '{}'));
@@ -393,10 +401,26 @@
if (d.sdk_int != null) lines.push('Android API: ' + d.sdk_int); if (d.sdk_int != null) lines.push('Android API: ' + d.sdk_int);
if (d.os_release) lines.push('Android version: ' + d.os_release); if (d.os_release) lines.push('Android version: ' + d.os_release);
if (d.abis && d.abis.length) lines.push('ABIs: ' + formatAbis(d.abis)); if (d.abis && d.abis.length) lines.push('ABIs: ' + formatAbis(d.abis));
if (d.wan_ip) lines.push('WAN IP (last poll): ' + d.wan_ip); if (d.lan_ip) lines.push('Device LAN IP: ' + d.lan_ip);
if (d.lan_ip) lines.push('LAN IP: ' + d.lan_ip); if (d.device_wan_ip) lines.push('Device WAN IP: ' + d.device_wan_ip);
if (d.poll_source_ip) {
let pollLabel = 'Poll source IP (server view)';
if (d.poll_source_is_private) {
pollLabel += ' — intra/private hop, not device WAN';
}
lines.push(pollLabel + ': ' + d.poll_source_ip);
}
if (d.vpn_ip) lines.push('VPN IP: ' + d.vpn_ip); if (d.vpn_ip) lines.push('VPN IP: ' + d.vpn_ip);
if (d.vpn_public_key) lines.push('VPN public key: ' + d.vpn_public_key); if (d.vpn_public_key) lines.push('VPN public key: ' + d.vpn_public_key);
if (d.wg_rx_bytes != null || d.wg_tx_bytes != null) {
const rx = formatBytes(d.wg_rx_bytes || 0);
const tx = formatBytes(d.wg_tx_bytes || 0);
lines.push('VPN traffic (BE wg): ↓' + rx + ' ↑' + tx);
}
if (d.wg_latest_handshake > 0) {
lines.push('VPN last handshake: ' + new Date(d.wg_latest_handshake * 1000).toISOString());
}
if (d.wg_endpoint) lines.push('VPN endpoint: ' + d.wg_endpoint);
lines.push('App version: ' + (d.app_version || '—')); lines.push('App version: ' + (d.app_version || '—'));
lines.push('Opt-in mode: ' + (d.opt_in_mode || 'none')); lines.push('Opt-in mode: ' + (d.opt_in_mode || 'none'));
lines.push('Last seen: ' + (d.last_seen_at || '—')); lines.push('Last seen: ' + (d.last_seen_at || '—'));

10
examples/crash_reporter/backend/scripts/verify_remote_access_prod.sh
View File

@@ -80,8 +80,14 @@ else
note "php CLI cannot run wg genkey — ensure www-data can run wg if provision_peers=true" note "php CLI cannot run wg genkey — ensure www-data can run wg if provision_peers=true"
fi fi
# DB tables (via bootstrap) # DB tables (via bootstrap) — use FPM user + php81 so PDO/mysql extensions match production
"$PHP" -r ' DB_RUN="$PHP"
DB_USER=""
if [[ "$(id -u)" -eq 0 ]] && id nginx >/dev/null 2>&1; then
DB_RUN="sudo -u nginx $PHP"
DB_USER="nginx"
fi
$DB_RUN -r '
require "'"$ROOT"'/src/bootstrap.php"; require "'"$ROOT"'/src/bootstrap.php";
RemoteAccessRepository::ensureSchema(); RemoteAccessRepository::ensureSchema();
$pdo = Database::pdo(); $pdo = Database::pdo();

51
examples/crash_reporter/backend/src/RemoteAccessRepository.php
View File

@@ -266,11 +266,12 @@ final class RemoteAccessRepository {
$pdo = Database::pdo(); $pdo = Database::pdo();
$now = self::nowSql(); $now = self::nowSql();
$pdo->prepare( $pdo->prepare(
'UPDATE remote_access_devices SET opt_in_mode = ?, last_random = ?, app_version = ?, last_seen_at = ?, updated_at = ? WHERE device_id = ?' 'UPDATE remote_access_devices SET whitelisted = 0, opt_in_mode = ?, last_random = ?, app_version = ?, last_seen_at = ?, updated_at = ? WHERE device_id = ?'
)->execute([self::OPT_NONE, $random, $appVersion, $now, $now, $deviceId]); )->execute([self::OPT_NONE, $random, $appVersion, $now, $now, $deviceId]);
self::closeActiveSessionsForDevice($deviceId, 'user_disabled'); self::closeActiveSessionsForDevice($deviceId, 'user_disabled');
self::logEvent($deviceId, null, null, 'user_disable', 'device_opt_out', null, $clientIp); self::logEvent($deviceId, null, null, 'user_disable', 'device_opt_out', null, $clientIp);
self::logEvent($deviceId, null, null, 'whitelist_revoked', 'device_opt_out', null, $clientIp);
return self::raEnvelope('disabled', ['reason' => 'user_opt_out']); return self::raEnvelope('disabled', ['reason' => 'user_opt_out']);
} }
@@ -667,7 +668,9 @@ final class RemoteAccessRepository {
$sql .= ' WHERE company_id = ?'; $sql .= ' WHERE company_id = ?';
$params[] = $companyId; $params[] = $companyId;
} }
$sql .= ' ORDER BY last_seen_at DESC, device_id ASC'; $sql .= Database::isMysql()
? ' ORDER BY last_seen_at DESC NULLS LAST, device_id ASC'
: ' ORDER BY datetime(COALESCE(last_seen_at, \'1970-01-01\')) DESC, device_id ASC';
$st = Database::pdo()->prepare($sql); $st = Database::pdo()->prepare($sql);
$st->execute($params); $st->execute($params);
return $st->fetchAll(PDO::FETCH_ASSOC) ?: []; return $st->fetchAll(PDO::FETCH_ASSOC) ?: [];
@@ -845,11 +848,17 @@ final class RemoteAccessRepository {
$row['os_release'] = $ctx['os_release'] ?? ''; $row['os_release'] = $ctx['os_release'] ?? '';
$row['sdk_int'] = $ctx['sdk_int'] ?? null; $row['sdk_int'] = $ctx['sdk_int'] ?? null;
$row['abis'] = $ctx['abis'] ?? []; $row['abis'] = $ctx['abis'] ?? [];
$row['wan_ip'] = self::resolveDeviceWanIp($row, $storedMeta);
$row['lan_ip'] = self::resolveDeviceLanIp($storedMeta); $row['lan_ip'] = self::resolveDeviceLanIp($storedMeta);
$row['poll_source_ip'] = self::resolvePollSourceIp($row);
$row['poll_source_is_private'] = self::isPrivateIp($row['poll_source_ip']);
$row['device_wan_ip'] = trim((string) ($storedMeta['wan_ip'] ?? ''));
$net = self::lookupDeviceSessionNetwork($deviceId); $net = self::lookupDeviceSessionNetwork($deviceId);
$row['vpn_ip'] = $net['vpn_ip'] ?? ''; $row['vpn_ip'] = $net['vpn_ip'] ?? '';
$row['vpn_public_key'] = $net['vpn_public_key'] ?? ''; $row['vpn_public_key'] = $net['vpn_public_key'] ?? '';
$row['wg_rx_bytes'] = $net['wg_rx_bytes'] ?? 0;
$row['wg_tx_bytes'] = $net['wg_tx_bytes'] ?? 0;
$row['wg_latest_handshake'] = $net['wg_latest_handshake'] ?? 0;
$row['wg_endpoint'] = $net['wg_endpoint'] ?? '';
$row['issue_count'] = $ctx['issue_count'] ?? 0; $row['issue_count'] = $ctx['issue_count'] ?? 0;
$row['graph_session_count'] = $ctx['graph_session_count'] ?? 0; $row['graph_session_count'] = $ctx['graph_session_count'] ?? 0;
$row['latest_issue_id'] = $ctx['latest_issue_id'] ?? null; $row['latest_issue_id'] = $ctx['latest_issue_id'] ?? null;
@@ -911,12 +920,8 @@ final class RemoteAccessRepository {
return (time() - $t) <= $maxAgeSec; return (time() - $t) <= $maxAgeSec;
} }
/** @param array<string, mixed> $storedMeta */ /** @param array<string, mixed> $row */
private static function resolveDeviceWanIp(array $row, array $storedMeta): string { private static function resolvePollSourceIp(array $row): string {
$fromDevice = trim((string) ($storedMeta['wan_ip'] ?? ''));
if ($fromDevice !== '') {
return self::normalizeClientIp($fromDevice);
}
$fromRow = trim((string) ($row['last_client_ip'] ?? '')); $fromRow = trim((string) ($row['last_client_ip'] ?? ''));
if ($fromRow !== '') { if ($fromRow !== '') {
return self::normalizeClientIp($fromRow); return self::normalizeClientIp($fromRow);
@@ -936,6 +941,17 @@ final class RemoteAccessRepository {
return self::normalizeClientIp($ip); return self::normalizeClientIp($ip);
} }
private static function isPrivateIp(string $ip): bool {
if ($ip === '' || !filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) {
return false;
}
if (!filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4 | FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE)) {
return true;
}
return false;
}
/** @param array<string, mixed> $storedMeta */ /** @param array<string, mixed> $storedMeta */
private static function resolveDeviceLanIp(array $storedMeta): string { private static function resolveDeviceLanIp(array $storedMeta): string {
return trim((string) ($storedMeta['lan_ip'] ?? '')); return trim((string) ($storedMeta['lan_ip'] ?? ''));
@@ -953,9 +969,16 @@ final class RemoteAccessRepository {
return $raw; return $raw;
} }
/** @return array{vpn_ip: string, vpn_public_key: string} */ /** @return array{vpn_ip: string, vpn_public_key: string, wg_rx_bytes: int, wg_tx_bytes: int, wg_latest_handshake: int, wg_endpoint: string} */
private static function lookupDeviceSessionNetwork(string $deviceId): array { private static function lookupDeviceSessionNetwork(string $deviceId): array {
$empty = ['vpn_ip' => '', 'vpn_public_key' => '']; $empty = [
'vpn_ip' => '',
'vpn_public_key' => '',
'wg_rx_bytes' => 0,
'wg_tx_bytes' => 0,
'wg_latest_handshake' => 0,
'wg_endpoint' => '',
];
if ($deviceId === '') { if ($deviceId === '') {
return $empty; return $empty;
} }
@@ -978,9 +1001,15 @@ final class RemoteAccessRepository {
$vpnIp = $addr !== '' ? preg_replace('/\/\d+$/', '', $addr) ?? $addr : ''; $vpnIp = $addr !== '' ? preg_replace('/\/\d+$/', '', $addr) ?? $addr : '';
$pub = trim((string) ($row['wg_client_public_key'] ?? '')); $pub = trim((string) ($row['wg_client_public_key'] ?? ''));
$stats = $pub !== '' ? WireGuardTrafficStats::forPublicKey($pub) : null;
return [ return [
'vpn_ip' => is_string($vpnIp) ? trim($vpnIp) : '', 'vpn_ip' => is_string($vpnIp) ? trim($vpnIp) : '',
'vpn_public_key' => $pub, 'vpn_public_key' => $pub,
'wg_rx_bytes' => (int) ($stats['rx_bytes'] ?? 0),
'wg_tx_bytes' => (int) ($stats['tx_bytes'] ?? 0),
'wg_latest_handshake' => (int) ($stats['latest_handshake'] ?? 0),
'wg_endpoint' => (string) ($stats['endpoint'] ?? ''),
]; ];
} }

73
examples/crash_reporter/backend/src/WireGuardTrafficStats.php Normal file
View File

@@ -0,0 +1,73 @@
<?php
declare(strict_types=1);
/** Live WireGuard transfer counters from `wg show dump` (BE data plane). */
final class WireGuardTrafficStats {
private function __construct() {}
/** @return array<string, array{rx_bytes: int, tx_bytes: int, latest_handshake: int, endpoint: string}> */
public static function peerMap(): array {
static $cache = null;
static $cacheAt = 0;
if ($cache !== null && (time() - $cacheAt) < 5) {
return $cache;
}
$cache = self::loadPeerMap();
$cacheAt = time();
return $cache;
}
/** @return array{rx_bytes: int, tx_bytes: int, latest_handshake: int, endpoint: string}|null */
public static function forPublicKey(string $publicKey): ?array {
$pub = trim($publicKey);
if ($pub === '') {
return null;
}
return self::peerMap()[$pub] ?? null;
}
/** @return array<string, array{rx_bytes: int, tx_bytes: int, latest_handshake: int, endpoint: string}> */
private static function loadPeerMap(): array {
$iface = (string) cfg('remote_access.wg_interface', 'wg0');
$prefix = rtrim((string) cfg('remote_access.wg_set_prefix', ''), " \t");
$cmd = ($prefix !== '' ? $prefix . ' ' : '') . 'wg show ' . escapeshellarg($iface) . ' dump 2>/dev/null';
$raw = @shell_exec($cmd);
if (!is_string($raw) || trim($raw) === '') {
return [];
}
$out = [];
foreach (preg_split('/\r\n|\n|\r/', trim($raw)) as $line) {
if ($line === '' || str_starts_with($line, 'interface:')) {
continue;
}
$cols = explode("\t", $line);
if (count($cols) < 7) {
continue;
}
$pub = trim($cols[0]);
if ($pub === '') {
continue;
}
$out[$pub] = [
'rx_bytes' => (int) ($cols[5] ?? 0),
'tx_bytes' => (int) ($cols[6] ?? 0),
'latest_handshake' => (int) ($cols[4] ?? 0),
'endpoint' => trim((string) ($cols[2] ?? '')),
];
}
return $out;
}
public static function formatBytes(int $bytes): string {
if ($bytes < 1024) {
return $bytes . ' B';
}
if ($bytes < 1024 * 1024) {
return round($bytes / 1024, 1) . ' KiB';
}
if ($bytes < 1024 * 1024 * 1024) {
return round($bytes / (1024 * 1024), 2) . ' MiB';
}
return round($bytes / (1024 * 1024 * 1024), 2) . ' GiB';
}
}

1
examples/crash_reporter/backend/src/bootstrap.php
View File

@@ -68,6 +68,7 @@ require_once __DIR__ . '/UrlShortenerDatabase.php';
require_once __DIR__ . '/ShortLinksRepository.php'; require_once __DIR__ . '/ShortLinksRepository.php';
require_once __DIR__ . '/WireGuardPeerProvisioner.php'; require_once __DIR__ . '/WireGuardPeerProvisioner.php';
require_once __DIR__ . '/WireGuardAddressPool.php'; require_once __DIR__ . '/WireGuardAddressPool.php';
require_once __DIR__ . '/WireGuardTrafficStats.php';
require_once __DIR__ . '/RsshSessionProvisioner.php'; require_once __DIR__ . '/RsshSessionProvisioner.php';
require_once __DIR__ . '/AnalyticsHead.php'; require_once __DIR__ . '/AnalyticsHead.php';

1
examples/crash_reporter/backend/views/layout.php
View File

@@ -443,6 +443,7 @@
Devices appear after they poll remote access (dev settings → WireGuard or RSSH). Sort is by <strong>last seen</strong> — top rows are actively polling. Devices appear after they poll remote access (dev settings → WireGuard or RSSH). Sort is by <strong>last seen</strong> — top rows are actively polling.
Whitelist rows marked <span class="tag-pill tag-pill--warn">Needs whitelist</span> (opted in, not yet allowed). Whitelist rows marked <span class="tag-pill tag-pill--warn">Needs whitelist</span> (opted in, not yet allowed).
Stale rows have not polled in 7+ days. Stale rows have not polled in 7+ days.
<strong>Poll source IP</strong> is the address BE sees on the HTTP request (often a router or proxy on <code>10.7.x.x</code>) — not the device WAN or LAN.
</p> </p>
<div class="reports-table-wrap"> <div class="reports-table-wrap">
<table class="data-table reports-tree reports-table--cols" id="ra-devices-table"> <table class="data-table reports-tree reports-table--cols" id="ra-devices-table">

268
examples/network/README.md
View File

@@ -1,204 +1,92 @@
# Edge network probes # Edge network watchdog
Scripts for **router**, **FE**, and **external vantage** (e.g. Raspberry Pi) — not BE app hosts. **One script:** `wan-watch.sh` — deploy everywhere, read one log, get a verdict every 5 minutes.
## Auto watchdog (v2.1) ## Deploy
When a cron snapshot exceeds thresholds, `wan-snapshot.sh` appends **alert** + **deep** lines to the same log (`WAN_WATCH_LOG`).
| `kind=` | Meaning |
|---------|---------|
| `snapshot` | Normal 15-min ping row |
| `alert` | Threshold breached; lists `triggers=` |
| `deep` | One parseable fact (`section=host`, `burst`, `iface`, `iptables`, …) |
**Deploy both files:**
```bash ```bash
sudo cp examples/network/wan-snapshot.sh examples/network/wan-deep-diag.sh /usr/local/bin/ sudo cp examples/network/wan-watch.sh /usr/local/bin/wan-watch.sh
sudo chmod +x /usr/local/bin/wan-snapshot.sh /usr/local/bin/wan-deep-diag.sh sudo chmod +x /usr/local/bin/wan-watch.sh
sudo cp examples/network/wan-watch.env.example /etc/conf.d/wan-watch
# edit WAN_WATCH_ROLE=pi|router|fe
``` ```
**Env** (`/etc/conf.d/wan-snapshot`): Remove old cron entries for `wan-snapshot.sh` / `monstro-netdiag.sh`. **One cron only:**
```cron
*/5 * * * * root . /etc/conf.d/wan-watch 2>/dev/null; /usr/local/bin/wan-watch.sh >>/var/log/wan-watch.cron.log 2>&1
```
Log: `/var/log/wan-watch.log`
## Every run (5 min) captures
| Layer | What |
|-------|------|
| **Ping matrix** | wan, fe, be, gw, router — loss/min/avg/max/mdev |
| **Verdict** | `kind=verdict` — immediate conclusion + action hint |
| **Tunnels** | GRE links/modules, PPTP (1723/pppd/pptpd), **tinc** (tincctl status), **WireGuard**, **iodine/dns0/tun0** |
| **Edge services** | named, squid, nginx, iodined — proc count + CPU |
| **Host** | load, cpu, conntrack, softnet drops, ss summary |
| **Network** | routes, neigh, iface drops, qdisc backlog, iptables hot rules (router) |
| **On anomaly** | `kind=alert` + `kind=deep` burst pings + full probe repeat (5 min cooldown) |
## Commands
```bash
wan-watch.sh # cron default
wan-watch.sh --full # instant human dump (when slow NOW)
wan-watch.sh --analyze # final conclusion from log (last 24h) — no waiting
wan-watch.sh --analyze 6 # last 6 hours
wan-watch.sh --timeline # hourly good/bad bars
wan-watch.sh --correlate /var/log/wan-watch.log path/to/other/log
wan-watch.sh --force-deep # full capture even if OK
wan-watch.sh --tail 30
```
## Verdicts (automatic)
| verdict | Meaning |
|---------|---------|
| `OK` | Within thresholds |
| `MONSTRO_WAN_BLOAT` | Router WAN slow, FE local OK — **your main issue** |
| `UPSTREAM_GW` | WAN + gw 10.7.16.228 both slow |
| `PATH_TO_FE` | Pi ISP OK, path to 10.7.0.10 slow |
| `CONNTRACK_PRESSURE` | nf_conntrack > 70% |
| `TUNNEL_DNS0_LOAD` | iodine dns0 drops elevated |
| `WAN_JITTER` | High mdev, bufferbloat signal |
Cron stdout example:
```text
WAN-WATCH [router/monstro] verdict=MONSTRO_WAN_BLOAT confidence=high summary="WAN leg slow..." action="SQM/fq_codel on xenbr0..."
```
## `--analyze` example output
```text
CONCLUSION: MONSTRO WAN EDGE QUEUEING (confidence=HIGH)
Fix order: 1) fq_codel/SQM on xenbr0 2) gw 10.7.16.228 3) squid/named 4) iodine dns0
```
Run this **any time** — no need to wait 24h after deploy.
## Host config (`/etc/conf.d/wan-watch`)
```sh ```sh
WAN_WATCH_LOG=/var/log/wan-watch.log WAN_WATCH_LOG=/var/log/wan-watch.log
WAN_SNAPSHOT_LOG=/var/log/wan-watch.log WAN_WATCH_ROLE=pi # or router on monstro
WAN_SNAPSHOT_WATCH=1 ROUTER_PING_TARGET=10.7.6.228 # Pi only
WAN_WATCH_DEEP=auto
WAN_WATCH_COOLDOWN_S=300
``` ```
**Triggers (defaults):** `*_avg_ms >= 50`, `*_loss >= 1%`, `mdev >= 80`, `conntrack >= 80%`, `neigh_fe` not REACHABLE/DELAY, plus role hints `pi_fe_path` / `router_wan_path`. ## Legacy
**Cooldown:** one deep capture per 14 min (`WAN_WATCH_COOLDOWN_S=840`) to avoid cron pile-up. - `wan-snapshot.sh` → wrapper to `wan-watch.sh`
- `wan-deep-diag.sh` → merged into `wan-watch.sh`
- `monstro-netdiag.sh` → use `wan-watch.sh --full` instead
**Review after a bad spell:** ## Known conclusion (from your logs)
```bash **MONSTRO WAN EDGE QUEUEING** on `xenbr0` → upstream `10.7.16.228`. Not FE/BE, not Pi ISP. Contributing load: **named**, **squid**, **dns0** (iodine). Fix: **SQM/fq_codel** on WAN iface first.
/usr/local/bin/wan-snapshot.sh --alerts 80
/usr/local/bin/wan-snapshot.sh --correlate /var/log/wan-watch.log …
grep 'kind=deep.*section=burst' /var/log/wan-watch.log | tail
grep 'kind=deep.*section=iptables' /var/log/wan-watch.log | tail
```
Example deep lines:
```text
kind=alert ts=2026-06-12T21:45:01+03:00 role=router host=monstro parent_ts=... triggers="wan_avg=535.365,router_wan_path"
kind=deep ts=2026-06-12T21:45:01+03:00 role=router host=monstro parent_ts=... section=burst label=wan target=1.1.1.1 loss=0% avg_ms=520 max_ms=790 mdev_ms=110
kind=deep ts=2026-06-12T21:45:01+03:00 role=router host=monstro parent_ts=... section=iptables tbl=filter chain=FORWARD line=3 pkts=188000 target=ACCEPT
```
## `wan-snapshot.sh` (v2)
Periodic ping comparison plus lightweight host/path context.
| Target | Default | Meaning |
|--------|---------|---------|
| **WAN** | `1.1.1.1` | ISP/upstream leg |
| **FE** | `10.7.0.10` | Path to Gentoo FE |
| **BE** | `10.7.16.128` | Deeper LAN (disable: `BE_PING_TARGET=none`) |
| **router** | `none` | Optional modem ping (`ROUTER_PING_TARGET` on Pi) |
| **gw** | auto | Default gateway ping |
Each target logs: `loss`, `min`, `avg`, `max`, `mdev` (jitter).
Extras (`WAN_SNAPSHOT_EXTRAS=1`): `load1`, `cpu_busy`, `conntrack`, `route_wan`/`route_fe`, `via_wan`, `neigh_fe`, non-zero `drops`.
Log format: `ver=2 role=pi|fe|router host=…` — older lines without `ver=` still work in `--report` / `--correlate`.
### Install (any host)
```bash
sudo cp examples/network/wan-snapshot.sh /usr/local/bin/wan-snapshot.sh
sudo chmod +x /usr/local/bin/wan-snapshot.sh
sudo cp examples/network/wan-snapshot.env.example /etc/conf.d/wan-snapshot
# edit /etc/conf.d/wan-snapshot for role-specific targets
```
### Cron (every 15 minutes — **one** entry per host)
```cron
*/15 * * * * root . /etc/conf.d/wan-snapshot 2>/dev/null; /usr/local/bin/wan-snapshot.sh >>/var/log/wan-quality.cron.log 2>&1
```
The script uses `flock` on `/var/run/wan-snapshot.lock` so duplicate crontab lines do not double-log.
Log: `/var/log/wan-quality.log` (override with `WAN_SNAPSHOT_LOG`).
### Suggested `/etc/conf.d/wan-snapshot`
**Raspberry Pi** (path into infra matters most):
```sh
WAN_SNAPSHOT_ROLE=pi
ROUTER_PING_TARGET=10.7.6.228
FE_PING_TARGET=10.7.0.10
BE_PING_TARGET=10.7.16.128
```
**Gentoo FE**:
```sh
WAN_SNAPSHOT_ROLE=fe
FE_PING_TARGET=10.7.0.10
BE_PING_TARGET=10.7.16.128
ROUTER_PING_TARGET=10.7.6.228
```
**Debian router**:
```sh
WAN_SNAPSHOT_ROLE=router
FE_PING_TARGET=10.7.0.10
BE_PING_TARGET=10.7.16.128
```
### Usage
```bash
# one line appended + printed
sudo /usr/local/bin/wan-snapshot.sh
# today vs yesterday (mean, p50, p90, bad>=50ms counts)
/usr/local/bin/wan-snapshot.sh --report
# recent samples
/usr/local/bin/wan-snapshot.sh --tail 48
# merge Pi + FE (+ router) logs by 15-min slot
/usr/local/bin/wan-snapshot.sh --correlate \
tmp/raspberrypi/var/log/wan-quality.log \
tmp/FE_gentoo/var/log/wan-quality.log
```
Example v2 line:
```text
ts=2026-06-12T15:30:01+03:00 ver=2 role=fe host=wg0 wan=1.1.1.1 wan_loss=0% wan_min_ms=9.1 wan_avg_ms=10.6 wan_max_ms=12.4 wan_mdev_ms=0.8 fe=10.7.0.10 fe_loss=0% fe_min_ms=0.05 fe_avg_ms=0.07 fe_max_ms=0.10 fe_mdev_ms=0.01 be=10.7.16.128 be_loss=0% be_avg_ms=0.20 load1=0.12 cpu_busy=4% route_wan=eno0 route_fe=lo neigh_fe=REACHABLE
```
### Reading results
| Pattern | Likely cause |
|---------|----------------|
| `wan` bad, `fe` good (on router) | ISP/upstream |
| `wan` + `fe` bad on router | modem CPU, conntrack, LAN |
| `fe` bad on **pi**, `fe` good on **fe** | path/tunnel to FE, not FE VM |
| `wan_max_ms` >> `wan_avg_ms` or high `mdev` | jitter / bufferbloat |
| `conntrack>80%` | edge table full |
| `neigh_fe` not `REACHABLE` | ARP/L2 toward FE |
| `route_fe=tun0` / `ppp0` on Pi | metric is tunnel quality |
| ping good, HTTPS slow | DNAT/nginx (not measured here) |
From **outside** LAN, still run occasionally:
```bash
mtr -rwzc 50 134.17.26.161
```
## `monstro-netdiag.sh` (bottleneck + iptables + processes)
Run on the **Debian edge router** (`10.7.6.228`) when infra feels stuck. Complements `wan-snapshot` with on-demand depth.
```bash
sudo cp examples/network/monstro-netdiag.sh /usr/local/bin/
sudo chmod +x /usr/local/bin/monstro-netdiag.sh
```
**Cron** (optional; pair with wan-snapshot on router only):
```cron
*/15 * * * * root /usr/local/bin/monstro-netdiag.sh --snapshot >>/var/log/monstro-netdiag.cron.log 2>&1
```
**When sluggish**:
```bash
sudo monstro-netdiag.sh --full
sudo monstro-netdiag.sh --watch 30
monstro-netdiag.sh --tail 48
```
### Why intermittent (excluding ISP)?
| Cause | What to look for |
|-------|------------------|
| **conntrack table full** | `conntrack>80%` in wan-snapshot or monstro |
| **Router CPU / softirq** | high `cpu_busy`, `softnet dropped` in `--full` |
| **iptables rule cost** | FORWARD/NAT rules with huge packet counts |
| **FE/BE VM host busy** | FE ping high from router, low `load1` on router |
| **ARP/route flap** | `neigh_fe` not REACHABLE |
| **Bufferbloat on LAN** | high `mdev` / `max`, 0% loss |
Good vs bad: save one `--full` output on a rocketship evening and diff when stuck.
### Safe checks on monstro (read-only)
```bash
cat /proc/sys/net/netfilter/nf_conntrack_count
cat /proc/sys/net/netfilter/nf_conntrack_max
iptables -L FORWARD -n -v --line-numbers | head
ip neigh show 10.7.0.10
```
Do **not** change iptables/NAT on monstro without a rollback plan (per infra policy).

698
examples/network/wan-snapshot.sh
View File

@@ -1,696 +1,4 @@
#!/bin/sh #!/bin/sh
# # Backward-compatible wrapper — canonical script is wan-watch.sh in the same directory.
# Log WAN vs LAN ping quality from any vantage (router, FE, Pi, …). D="$(CDPATH= cd -- "$(dirname "$0")" && pwd)"
# Use to compare "rocketship" evenings vs slow periods (ISP/upstream vs local). exec "$D/wan-watch.sh" "$@"
#
# Topology (see docs/INFRA.md):
# world -> 134.17.26.161 (router 10.7.6.228) -> 10.7.0.10 (FE) -> 10.7.0.0/8 (BE, …)
#
# Install:
# sudo cp wan-snapshot.sh /usr/local/bin/wan-snapshot.sh
# sudo chmod +x /usr/local/bin/wan-snapshot.sh
# sudo mkdir -p /var/log
#
# Cron (every 15 minutes — ONE entry only; script uses flock to skip overlaps):
# */15 * * * * root /usr/local/bin/wan-snapshot.sh >>/var/log/wan-quality.cron.log 2>&1
#
# Env overrides:
# WAN_WATCH_LOG=/var/log/wan-watch.log (unified: snapshot + alert + deep)
# WAN_SNAPSHOT_LOG=/var/log/wan-quality.log (legacy snapshot copy if different)
# WAN_SNAPSHOT_ROLE=auto|router|fe|pi
# WAN_SNAPSHOT_EXTRAS=1
# WAN_SNAPSHOT_WATCH=1 (0 = disable auto deep-diag on anomaly)
# WAN_SNAPSHOT_LOCK=/var/run/wan-snapshot.lock
# WAN_WATCH_COOLDOWN_S=840 (min seconds between deep captures)
# WAN_ALERT_MS=50 FE_ALERT_MS=50 BE_ALERT_MS=50 GW_ALERT_MS=50
# WAN_LOSS_ALERT_PCT=1 WAN_MDEV_ALERT_MS=80 WAN_CONNTRACK_ALERT_PCT=80
# WAN_PING_TARGET=1.1.1.1 FE_PING_TARGET=10.7.0.10
# BE_PING_TARGET=10.7.16.128 ROUTER_PING_TARGET=none GW_PING_TARGET=auto
# PING_COUNT=10 PING_DEADLINE_S=15
#
# Install deep-diag library next to this script:
# cp wan-snapshot.sh wan-deep-diag.sh /usr/local/bin/
#
set -eu
WAN_SNAPSHOT_LOG="${WAN_SNAPSHOT_LOG:-/var/log/wan-quality.log}"
WAN_WATCH_LOG="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
WAN_SNAPSHOT_ROLE="${WAN_SNAPSHOT_ROLE:-auto}"
WAN_SNAPSHOT_EXTRAS="${WAN_SNAPSHOT_EXTRAS:-1}"
WAN_SNAPSHOT_WATCH="${WAN_SNAPSHOT_WATCH:-1}"
WAN_SNAPSHOT_LOCK="${WAN_SNAPSHOT_LOCK:-/var/run/wan-snapshot.lock}"
WAN_WATCH_COOLDOWN_S="${WAN_WATCH_COOLDOWN_S:-840}"
WAN_WATCH_COOLDOWN_FILE="${WAN_WATCH_COOLDOWN_FILE:-/var/run/wan-watch-deep.stamp}"
WAN_ALERT_MS="${WAN_ALERT_MS:-50}"
FE_ALERT_MS="${FE_ALERT_MS:-50}"
BE_ALERT_MS="${BE_ALERT_MS:-50}"
GW_ALERT_MS="${GW_ALERT_MS:-50}"
ROUTER_ALERT_MS="${ROUTER_ALERT_MS:-50}"
WAN_LOSS_ALERT_PCT="${WAN_LOSS_ALERT_PCT:-1}"
WAN_MDEV_ALERT_MS="${WAN_MDEV_ALERT_MS:-80}"
WAN_CONNTRACK_ALERT_PCT="${WAN_CONNTRACK_ALERT_PCT:-80}"
WAN_PING_TARGET="${WAN_PING_TARGET:-1.1.1.1}"
FE_PING_TARGET="${FE_PING_TARGET:-10.7.0.10}"
BE_PING_TARGET="${BE_PING_TARGET-10.7.16.128}"
ROUTER_PING_TARGET="${ROUTER_PING_TARGET:-none}"
GW_PING_TARGET="${GW_PING_TARGET:-auto}"
PING_COUNT="${PING_COUNT:-10}"
PING_DEADLINE_S="${PING_DEADLINE_S:-15}"
_WAN_SNAPSHOT_DIR="$(CDPATH= cd -- "$(dirname "$0")" && pwd)"
# shellcheck source=/dev/null
. "$_WAN_SNAPSHOT_DIR/wan-deep-diag.sh" 2>/dev/null \
|| . /usr/local/bin/wan-deep-diag.sh 2>/dev/null \
|| . /usr/local/lib/wan-deep-diag.sh 2>/dev/null \
|| true
usage() {
sed -n '2,28p' "$0"
printf '\nCommands:\n'
printf ' (default) append one snapshot line to the log\n'
printf ' --report summarize today vs yesterday\n'
printf ' --tail [N] print last N log lines (default 24)\n'
printf ' --correlate FILE… merge logs by 15-min slot (role= or path basename)\n'
printf ' --alerts [N] recent kind=alert + kind=deep lines (default 40)\n'
printf ' --help this text\n'
}
need_ping() {
command -v ping >/dev/null 2>&1 || {
echo "wan-snapshot: missing ping" >&2
exit 1
}
}
detect_role() {
_role="$WAN_SNAPSHOT_ROLE"
if [ "$_role" != "auto" ]; then
printf '%s' "$_role"
return 0
fi
_hn="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)"
case "$_hn" in
*raspberry*|*rpi*) printf 'pi' ;;
*monstro*|*router*|*gw*) printf 'router' ;;
*gentoo*|*wg0*|*fe*) printf 'fe' ;;
*) printf 'host' ;;
esac
}
# ping_summary HOST -> loss_pct min_ms avg_ms max_ms mdev_ms (-1 for missing)
ping_summary() {
_host="$1"
_out=""
_out="$(ping -c "$PING_COUNT" -w "$PING_DEADLINE_S" "$_host" 2>/dev/null)" || true
if [ -z "$_out" ]; then
printf '%s %s %s %s %s' "-1" "-1" "-1" "-1" "-1"
return 0
fi
_loss="$(printf '%s\n' "$_out" | awk '/packet loss/ {
for (i = 1; i <= NF; i++) if ($i ~ /%$/) { gsub(/%/, "", $i); print $i; exit }
}')"
_min="-1"
_avg="-1"
_max="-1"
_mdev="-1"
_rtt="$(printf '%s\n' "$_out" | sed -n 's/.*= \([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\) ms.*/\1 \2 \3 \4/p' | head -1)"
if [ -n "$_rtt" ]; then
_min="${_rtt%% *}"
_rest="${_rtt#* }"
_avg="${_rest%% *}"
_rest="${_rest#* }"
_max="${_rest%% *}"
_mdev="${_rest#* }"
else
_avg="$(printf '%s\n' "$_out" | awk -F'[=/ ]' '/min\/avg\/max/ { print $8; exit }')"
[ -n "$_avg" ] || _avg="-1"
fi
[ -n "$_loss" ] || _loss="-1"
printf '%s %s %s %s %s' "$_loss" "$_min" "$_avg" "$_max" "$_mdev"
}
append_ping_fields() {
_line="$1"
_prefix="$2"
_host="$3"
_sum="$(ping_summary "$_host")"
_loss="${_sum%% *}"
_rest="${_sum#* }"
_min="${_rest%% *}"
_rest="${_rest#* }"
_avg="${_rest%% *}"
_rest="${_rest#* }"
_max="${_rest%% *}"
_mdev="${_rest#* }"
printf '%s %s=%s %s_loss=%s%% %s_min_ms=%s %s_avg_ms=%s %s_max_ms=%s %s_mdev_ms=%s' \
"$_line" "$_prefix" "$_host" "$_prefix" "$_loss" "$_prefix" "$_min" \
"$_prefix" "$_avg" "$_prefix" "$_max" "$_prefix" "$_mdev"
}
route_dev() {
_host="$1"
ip route get "$_host" 2>/dev/null | awk '{for (i = 1; i <= NF; i++) if ($i == "dev") { print $(i + 1); exit }}'
}
route_gw() {
_host="$1"
ip route get "$_host" 2>/dev/null | awk '{for (i = 1; i <= NF; i++) if ($i == "via") { print $(i + 1); exit }}'
}
default_gw() {
ip route show default 2>/dev/null | awk '{print $3; exit}'
}
load_1m() {
awk '{print $1}' /proc/loadavg 2>/dev/null || echo "n/a"
}
cpu_busy_pct() {
awk '/^cpu / {
idle = $5 + $6; total = 0
for (i = 2; i <= NF; i++) total += $i
if (total > 0) printf "%.0f", 100 * (total - idle) / total
}' /proc/stat 2>/dev/null || echo "n/a"
}
conntrack_pct() {
_cur="$(cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null)" || _cur=""
_max="$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null)" || _max=""
if [ -n "$_cur" ] && [ -n "$_max" ] && [ "$_max" -gt 0 ] 2>/dev/null; then
_pct=$((_cur * 100 / _max))
printf 'conntrack=%s/%s(%s%%)' "$_cur" "$_max" "$_pct"
fi
}
neigh_state() {
_host="$1"
ip neigh show "$_host" 2>/dev/null | awk '{print $6; exit}'
}
iface_drops_compact() {
ip -s link 2>/dev/null | awk '
/^[0-9]+:/ {
if (iface != "" && iface !~ /^lo$/) {
if (rx_drop + tx_drop + rx_err + tx_err > 0)
printf "%s:rd=%d,td=%d,re=%d,te=%d ", iface, rx_drop, tx_drop, rx_err, tx_err
}
iface = $2; gsub(":", "", iface)
rx_drop = tx_drop = rx_err = tx_err = 0
}
/RX:/ { getline; rx_err = $3; rx_drop = $4 }
/TX:/ { getline; tx_err = $3; tx_drop = $4 }
END {
if (iface != "" && iface !~ /^lo$/ && rx_drop + tx_drop + rx_err + tx_err > 0)
printf "%s:rd=%d,td=%d,re=%d,te=%d", iface, rx_drop, tx_drop, rx_err, tx_err
}
' | sed 's/ $//'
}
# True if line is a snapshot row (legacy ts=… or kind=snapshot).
is_snapshot_line() {
case "$1" in
kind=snapshot*) return 0 ;;
ts=*) return 0 ;;
*) return 1 ;;
esac
}
num_ge() {
_a="$1"
_b="$2"
[ "$_a" != "-1" ] && [ -n "$_a" ] && awk -v a="$_a" -v b="$_b" 'BEGIN { exit (a + 0 >= b + 0) ? 0 : 1 }'
}
watch_cooldown_active() {
[ -r "$WAN_WATCH_COOLDOWN_FILE" ] || return 1
_last="$(cat "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null)" || return 1
_now="$(date +%s 2>/dev/null)" || return 1
[ "$((_now - _last))" -lt "$WAN_WATCH_COOLDOWN_S" ] 2>/dev/null
}
watch_cooldown_stamp() {
date +%s > "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null || true
}
# Returns trigger list (comma-separated) or empty.
detect_anomalies() {
_line="$1"
_role="$2"
_triggers=""
for _pfx in wan fe be gw router; do
_avg="$(field_value "$_line" "${_pfx}_avg_ms")"
_loss="$(field_value "$_line" "${_pfx}_loss")"
_mdev="$(field_value "$_line" "${_pfx}_mdev_ms")"
_max="$(field_value "$_line" "${_pfx}_max_ms")"
_thr="$FE_ALERT_MS"
case "$_pfx" in
wan) _thr="$WAN_ALERT_MS" ;;
fe) _thr="$FE_ALERT_MS" ;;
be) _thr="$BE_ALERT_MS" ;;
gw) _thr="$GW_ALERT_MS" ;;
router) _thr="$ROUTER_ALERT_MS" ;;
esac
if num_ge "$_avg" "$_thr"; then
_triggers="${_triggers}${_triggers:+,}${_pfx}_avg=${_avg}"
fi
if num_ge "$_loss" "$WAN_LOSS_ALERT_PCT"; then
_triggers="${_triggers}${_triggers:+,}${_pfx}_loss=${_loss}"
fi
if num_ge "$_mdev" "$WAN_MDEV_ALERT_MS"; then
_triggers="${_triggers}${_triggers:+,}${_pfx}_mdev=${_mdev}"
fi
if num_ge "$_max" "$((_thr * 3))"; then
_triggers="${_triggers}${_triggers:+,}${_pfx}_max=${_max}"
fi
done
_neigh="$(printf '%s\n' "$_line" | sed -n 's/.*neigh_fe=\([^ ]*\).*/\1/p')"
if [ -n "$_neigh" ] && [ "$_neigh" != "REACHABLE" ] && [ "$_neigh" != "DELAY" ]; then
_triggers="${_triggers}${_triggers:+,}neigh_fe=${_neigh}"
fi
_ct_pct="$(printf '%s\n' "$_line" | sed -n 's/.*(\([0-9]*\)%).*/\1/p' | head -1)"
if [ -n "$_ct_pct" ] && num_ge "$_ct_pct" "$WAN_CONNTRACK_ALERT_PCT"; then
_triggers="${_triggers}${_triggers:+,}conntrack_pct=${_ct_pct}"
fi
# Pi: WAN ok but FE slow -> path anomaly (even if FE just above threshold).
if [ "$_role" = "pi" ]; then
_wan_avg="$(field_value "$_line" "wan_avg_ms")"
_fe_avg="$(field_value "$_line" "fe_avg_ms")"
if num_ge "$_fe_avg" "$FE_ALERT_MS" && ! num_ge "$_wan_avg" "$WAN_ALERT_MS"; then
case "$_triggers" in *fe_avg=*) ;; *)
_triggers="${_triggers}${_triggers:+,}pi_fe_path" ;;
esac
fi
fi
# Router: WAN bad, FE ok -> edge WAN leg (matches monstro xenbr0 pattern).
if [ "$_role" = "router" ]; then
_wan_avg="$(field_value "$_line" "wan_avg_ms")"
_fe_avg="$(field_value "$_line" "fe_avg_ms")"
if num_ge "$_wan_avg" "$WAN_ALERT_MS" && ! num_ge "$_fe_avg" "$FE_ALERT_MS"; then
case "$_triggers" in *wan_avg=*) ;; *)
_triggers="${_triggers}${_triggers:+,}router_wan_path" ;;
esac
fi
fi
printf '%s' "$_triggers"
}
maybe_deep_diag() {
_line="$1"
_ts="$2"
_role="$3"
_hostn="$4"
[ "$WAN_SNAPSHOT_WATCH" = "1" ] || return 0
[ "${WAN_DEEP_DIAG_LOADED:-0}" = "1" ] || {
echo "wan-snapshot: watch enabled but wan-deep-diag.sh not loaded (install alongside wan-snapshot.sh)" >&2
return 0
}
_triggers="$(detect_anomalies "$_line" "$_role")"
[ -n "$_triggers" ] || return 0
if watch_cooldown_active; then
wan_deep_emit alert "$_ts" "$_role" "$_hostn" \
"parent_ts=$_ts triggers=\"$_triggers\" action=skipped_cooldown"
return 0
fi
_gw_target="$GW_PING_TARGET"
if [ "$_gw_target" = "auto" ]; then
_gw_target="$(default_gw)"
fi
_wdev="$(route_dev "$WAN_PING_TARGET")"
_fdev="$(route_dev "$FE_PING_TARGET")"
wan_deep_diag_run "$_ts" "$_role" "$_hostn" "$_triggers" \
"$WAN_PING_TARGET" "$FE_PING_TARGET" "$_gw_target" "$_wdev" "$_fdev"
watch_cooldown_stamp
}
append_watch_log() {
_line="$1"
_dir="$(dirname "$WAN_WATCH_LOG")"
[ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true
printf '%s\n' "$_line" >> "$WAN_WATCH_LOG"
}
acquire_lock() {
command -v flock >/dev/null 2>&1 || return 0
_lock="$WAN_SNAPSHOT_LOCK"
_dir="$(dirname "$_lock")"
_fallback="${TMPDIR:-/tmp}/wan-snapshot.$(id -u 2>/dev/null || echo 0).lock"
if [ ! -d "$_dir" ] || [ ! -w "$_dir" ] 2>/dev/null; then
_lock="$_fallback"
elif ! ( : >>"$_lock" ) 2>/dev/null; then
_lock="$_fallback"
fi
exec 9>"$_lock" 2>/dev/null || return 0
flock -n 9 || {
echo "wan-snapshot: skip (already running, lock=$_lock)" >&2
exit 0
}
}
snapshot_once() {
need_ping
acquire_lock
_ts="$(date -Iseconds 2>/dev/null || date '+%Y-%m-%dT%H:%M:%S%z')"
_role="$(detect_role)"
_host="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)"
_line="kind=snapshot ts=$_ts ver=2 role=$_role host=$_host"
_line="$(append_ping_fields "$_line" "wan" "$WAN_PING_TARGET")"
_line="$(append_ping_fields "$_line" "fe" "$FE_PING_TARGET")"
if [ -n "$BE_PING_TARGET" ] && [ "$BE_PING_TARGET" != "none" ]; then
_line="$(append_ping_fields "$_line" "be" "$BE_PING_TARGET")"
fi
if [ -n "$ROUTER_PING_TARGET" ] && [ "$ROUTER_PING_TARGET" != "none" ]; then
_line="$(append_ping_fields "$_line" "router" "$ROUTER_PING_TARGET")"
fi
_gw_target="$GW_PING_TARGET"
if [ "$_gw_target" = "auto" ]; then
_gw_target="$(default_gw)"
[ -n "$_gw_target" ] || _gw_target="none"
fi
if [ -n "$_gw_target" ] && [ "$_gw_target" != "none" ]; then
_line="$(append_ping_fields "$_line" "gw" "$_gw_target")"
fi
if [ "$WAN_SNAPSHOT_EXTRAS" = "1" ]; then
_line="$_line load1=$(load_1m) cpu_busy=$(cpu_busy_pct)%"
_ct="$(conntrack_pct)"
[ -n "$_ct" ] && _line="$_line $_ct"
_wdev="$(route_dev "$WAN_PING_TARGET")"
_fdev="$(route_dev "$FE_PING_TARGET")"
[ -n "$_wdev" ] && _line="$_line route_wan=$_wdev"
[ -n "$_fdev" ] && _line="$_line route_fe=$_fdev"
_wgw="$(route_gw "$WAN_PING_TARGET")"
[ -n "$_wgw" ] && _line="$_line via_wan=$_wgw"
_nfe="$(neigh_state "$FE_PING_TARGET")"
[ -n "$_nfe" ] && _line="$_line neigh_fe=$_nfe"
_drops="$(iface_drops_compact)"
[ -n "$_drops" ] && _line="$_line drops=\"$_drops\""
fi
append_watch_log "$_line"
if [ "$WAN_SNAPSHOT_LOG" != "$WAN_WATCH_LOG" ]; then
_dir="$(dirname "$WAN_SNAPSHOT_LOG")"
[ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true
printf '%s\n' "$_line" >> "$WAN_SNAPSHOT_LOG"
fi
maybe_deep_diag "$_line" "$_ts" "$_role" "$_host"
printf '%s\n' "$_line"
}
# Extract numeric field from log line (supports wan_avg_ms and legacy layouts).
field_value() {
_line="$1"
_field="$2"
printf '%s\n' "$_line" | awk -v f="$_field" '
{
for (i = 1; i <= NF; i++) {
if ($i ~ "^" f "=") {
split($i, a, "=")
v = a[2]
gsub(/%/, "", v)
if (v + 0 == v) { print v; exit }
}
}
}
'
}
day_stats() {
_log="$1"
_date="$2"
_field="$3"
[ -r "$_log" ] || return 0
awk -v d="$_date" -v f="$_field" '
($0 ~ "^kind=snapshot ts=" d || ($0 ~ "^ts=" d && $0 !~ / kind=/)) {
for (i = 1; i <= NF; i++) {
if ($i ~ "^" f "=") {
split($i, a, "=")
v = a[2]
gsub(/%/, "", v)
if (v + 0 == v && v >= 0) {
n++; s += v; if (v > max) max = v
if (v >= 50) bad++
vals[n] = v
}
}
}
}
END {
if (n == 0) { print "n/a n/a n/a n/a"; exit }
asort(vals)
p50 = vals[int((n + 1) * 0.5)]
p90 = vals[int(n * 0.9)]
if (p90 == "") p90 = vals[n]
printf "%.2f %.2f %.2f %.0f %d/%d", s / n, p50, p90, max, bad + 0, n
}
' "$_log"
}
day_max_loss() {
_log="$1"
_date="$2"
_field="$3"
[ -r "$_log" ] || return 0
awk -v d="$_date" -v f="$_field" '
($0 ~ "^kind=snapshot ts=" d || ($0 ~ "^ts=" d && $0 !~ / kind=/)) {
for (i = 1; i <= NF; i++) {
if ($i ~ "^" f "=") {
split($i, a, "=")
v = a[2]
gsub(/%/, "", v)
if (v + 0 == v && v >= 0 && v > max) max = v
}
}
}
END {
if (max == "") print "n/a"
else printf "%.0f", max
}
' "$_log"
}
report_day_block() {
_log="$1"
_label="$2"
_date="$3"
_samples="$(grep -cE "^kind=snapshot ts=$_date|^ts=$_date" "$_log" 2>/dev/null || echo 0)"
printf '%s=%s samples=%s\n' "$_label" "$_date" "$_samples"
for _pfx in wan fe be router gw; do
_stats="$(day_stats "$_log" "$_date" "${_pfx}_avg_ms")"
_mean="${_stats%% *}"
_rest="${_stats#* }"
_p50="${_rest%% *}"
_rest="${_rest#* }"
_p90="${_rest%% *}"
_rest="${_rest#* }"
_max="${_rest%% *}"
_bad="${_rest#* }"
_loss="$(day_max_loss "$_log" "$_date" "${_pfx}_loss")"
if [ "$_mean" != "n/a" ]; then
printf ' %s: mean=%s p50=%s p90=%s max=%s bad(>=50ms)=%s loss_max=%s%%\n' \
"$_pfx" "$_mean" "$_p50" "$_p90" "$_max" "$_bad" "$_loss"
fi
done
}
report() {
_log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
_today="$(date '+%Y-%m-%d')"
_yesterday="$(date -d 'yesterday' '+%Y-%m-%d' 2>/dev/null || date -v-1d '+%Y-%m-%d' 2>/dev/null || echo '')"
if [ ! -r "$_log" ]; then
echo "wan-snapshot: no log at $_log (run default command first)" >&2
exit 1
fi
_role="$(detect_role)"
printf 'log=%s role=%s\n' "$_log" "$_role"
report_day_block "$_log" "today" "$_today"
if [ -n "$_yesterday" ] && grep -qE "^kind=snapshot ts=$_yesterday|^ts=$_yesterday" "$_log" 2>/dev/null; then
report_day_block "$_log" "yesterday" "$_yesterday"
fi
printf '\nInterpretation:\n'
printf ' wan bad + fe good -> ISP/upstream or route_dev issue\n'
printf ' wan + fe bad -> router/LAN/tunnel path\n'
printf ' fe bad (pi) + fe ok (fe) -> path to FE, not FE host CPU\n'
printf ' high mdev / max vs avg -> jitter (bufferbloat, conntrack, VPN)\n'
printf ' conntrack>80%% -> edge table pressure (router)\n'
printf ' neigh_fe not REACHABLE -> ARP/L2 flap toward FE\n'
printf ' ping good, HTTPS slow -> DNAT/nginx above this probe\n'
}
tail_log() {
_n="${1:-24}"
_log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
[ -r "$_log" ] || {
echo "wan-snapshot: no log at $_log" >&2
exit 1
}
grep -E '^kind=snapshot |^ts=' "$_log" 2>/dev/null | tail -n "$_n"
}
tail_alerts() {
_n="${1:-40}"
_log="${WAN_WATCH_LOG:-$WAN_SNAPSHOT_LOG}"
[ -r "$_log" ] || {
echo "wan-snapshot: no log at $_log" >&2
exit 1
}
grep -E '^kind=alert |^kind=deep ' "$_log" 2>/dev/null | tail -n "$_n"
}
# Guess role label for correlate when log line has no role=.
log_role_label() {
_path="$1"
_line="$2"
_role="$(field_value "$_line" "role")"
if [ -n "$_role" ] && [ "$_role" != "0" ]; then
printf '%s' "$_role"
return 0
fi
case "$_path" in
*raspberry*|*rpi*) printf 'pi' ;;
*FE_gentoo*|*gentoo*) printf 'fe' ;;
*monstro*|*router*) printf 'router' ;;
*) printf '%s' "$(basename "$_path" .log)" ;;
esac
}
# Bucket timestamp to 15-min wall-clock slot: YYYY-MM-DDTHH:00|15|30|45
slot_key() {
_ts="$1"
_date="${_ts%%T*}"
_rest="${_ts#*T}"
_hour="${_rest%%:*}"
_min="${_rest#*:}"
_min="${_min%%:*}"
_min="${_min%%+*}"
_min="${_min%%-*}"
_bucket=$(( (_min / 15) * 15 ))
printf '%sT%s:%02d\n' "$_date" "$_hour" "$_bucket"
}
correlate_logs() {
if [ "$#" -lt 1 ]; then
echo "wan-snapshot: --correlate needs at least one log file" >&2
exit 1
fi
_tmp="$(mktemp "${TMPDIR:-/tmp}/wan-correlate.XXXXXX")"
# shellcheck disable=SC2064
trap 'rm -f "$_tmp"' EXIT INT HUP
for _path in "$@"; do
[ -r "$_path" ] || {
echo "wan-snapshot: unreadable: $_path" >&2
continue
}
_label="$(basename "$_path")"
while IFS= read -r _line || [ -n "$_line" ]; do
is_snapshot_line "$_line" || continue
case "$_line" in
kind=snapshot*) _ts="${_line#kind=snapshot ts=}" ;;
ts=*) _ts="${_line#ts=}" ;;
*) continue ;;
esac
_ts="${_ts%% *}"
_slot="$(slot_key "$_ts")"
_role="$(log_role_label "$_path" "$_line")"
printf '%s\t%s\t%s\t%s\t%s\n' "$_slot" "$_role" "$_ts" "$_label" "$_line" >> "$_tmp"
done < "$_path"
done
printf 'slot\t'
_roles="$(awk -F'\t' '{print $2}' "$_tmp" | sort -u | tr '\n' ' ')"
for _role in $_roles; do
printf '%s_wan\t%s_fe\t%s_be\t' "$_role" "$_role" "$_role"
done
printf '\n'
awk -F'\t' '
{
slot = $1; role = $2; ts = $3; line = $5
key = slot SUBSEP role
if (!(key in best_ts) || ts > best_ts[key]) {
best_ts[key] = ts
wan = fe = be = "-"
for (i = 1; i <= split(line, a, " "); i++) {
if (a[i] ~ /^wan_avg_ms=/) { split(a[i], b, "="); wan = b[2] }
if (a[i] ~ /^fe_avg_ms=/) { split(a[i], b, "="); fe = b[2] }
if (a[i] ~ /^be_avg_ms=/) { split(a[i], b, "="); be = b[2] }
}
W[slot, role, "wan"] = wan
W[slot, role, "fe"] = fe
W[slot, role, "be"] = be
slots[slot] = 1
roles[role] = 1
}
}
END {
nroles = 0
for (r in roles) rolelist[++nroles] = r
asort(rolelist)
nslots = 0
for (s in slots) slotlist[++nslots] = s
asort(slotlist)
for (si = 1; si <= nslots; si++) {
s = slotlist[si]
printf "%s", s
for (ri = 1; ri <= nroles; ri++) {
r = rolelist[ri]
printf "\t%s\t%s\t%s", W[s, r, "wan"] + 0 ? W[s, r, "wan"] : "-", \
W[s, r, "fe"] + 0 ? W[s, r, "fe"] : "-", \
W[s, r, "be"] + 0 ? W[s, r, "be"] : "-"
}
printf "\n"
}
}
' "$_tmp" | while IFS= read -r _row; do
printf '%s\n' "$_row"
done
}
case "${1:-}" in
-h|--help|help)
usage
;;
--report|report)
report
;;
--tail|tail)
tail_log "${2:-24}"
;;
--alerts|alerts)
tail_alerts "${2:-40}"
;;
--correlate|correlate)
shift
correlate_logs "$@"
;;
"")
snapshot_once
;;
*)
echo "wan-snapshot: unknown command: $1 (try --help)" >&2
exit 1
;;
esac

30
examples/network/wan-watch.env.example Normal file
View File

@@ -0,0 +1,30 @@
# /etc/conf.d/wan-watch — source from cron on Pi, monstro, FE
#
# Deploy ONE file:
# sudo cp wan-watch.sh /usr/local/bin/wan-watch.sh && sudo chmod +x /usr/local/bin/wan-watch.sh
#
# Cron (every 5 minutes):
# */5 * * * * root . /etc/conf.d/wan-watch 2>/dev/null; /usr/local/bin/wan-watch.sh >>/var/log/wan-watch.cron.log 2>&1
WAN_WATCH_LOG=/var/log/wan-watch.log
WAN_WATCH_ROLE=pi # pi | router | fe
# Pi
# ROUTER_PING_TARGET=10.7.6.228
# FE_PING_TARGET=10.7.0.10
# BE_PING_TARGET=10.7.16.128
# monstro
# WAN_WATCH_ROLE=router
# FE
# WAN_WATCH_ROLE=fe
# ROUTER_PING_TARGET=10.7.6.228
WAN_WATCH_DEEP=auto # auto | always | never
WAN_WATCH_COOLDOWN_S=300 # 5 min between deep captures
# Thresholds (ms)
# WAN_ALERT_MS=50
# FE_ALERT_MS=50
# CONNTRACK_ALERT_PCT=70

671
examples/network/wan-watch.sh Executable file
View File

@@ -0,0 +1,671 @@
#!/bin/sh
#
# wan-watch.sh — one script, one log, immediate verdict every run.
#
# Deploy (Pi + monstro + FE):
# sudo cp wan-watch.sh /usr/local/bin/wan-watch.sh
# sudo chmod +x /usr/local/bin/wan-watch.sh
# sudo cp wan-watch.env.example /etc/conf.d/wan-watch
#
# Cron (every 5 min — fast enough to catch 15-min pulses):
# */5 * * * * root . /etc/conf.d/wan-watch 2>/dev/null; /usr/local/bin/wan-watch.sh >>/var/log/wan-watch.cron.log 2>&1
#
# Manual:
# wan-watch.sh # cron snapshot + probes + verdict
# wan-watch.sh --full # human-readable dump NOW (no wait)
# wan-watch.sh --analyze # conclusion from log (last 24h default)
# wan-watch.sh --timeline # hourly good/bad bars
# wan-watch.sh --force-deep # full deep capture even if OK
#
set -eu
WAN_WATCH_LOG="${WAN_WATCH_LOG:-/var/log/wan-watch.log}"
WAN_WATCH_ROLE="${WAN_WATCH_ROLE:-${WAN_SNAPSHOT_ROLE:-auto}}"
WAN_WATCH_LOCK="${WAN_WATCH_LOCK:-/var/run/wan-watch.lock}"
WAN_WATCH_COOLDOWN_S="${WAN_WATCH_COOLDOWN_S:-300}"
WAN_WATCH_COOLDOWN_FILE="${WAN_WATCH_COOLDOWN_FILE:-/var/run/wan-watch-deep.stamp}"
WAN_WATCH_DEEP="${WAN_WATCH_DEEP:-auto}"
WAN_ANALYZE_HOURS="${WAN_ANALYZE_HOURS:-24}"
WAN_PING_TARGET="${WAN_PING_TARGET:-1.1.1.1}"
FE_PING_TARGET="${FE_PING_TARGET:-10.7.0.10}"
BE_PING_TARGET="${BE_PING_TARGET-10.7.16.128}"
ROUTER_PING_TARGET="${ROUTER_PING_TARGET:-none}"
GW_PING_TARGET="${GW_PING_TARGET:-auto}"
PING_COUNT="${PING_COUNT:-3}"
PING_DEADLINE_S="${PING_DEADLINE_S:-6}"
DEEP_PING_COUNT="${DEEP_PING_COUNT:-8}"
DEEP_PING_DEADLINE_S="${DEEP_PING_DEADLINE_S:-10}"
WAN_ALERT_MS="${WAN_ALERT_MS:-50}"
FE_ALERT_MS="${FE_ALERT_MS:-50}"
BE_ALERT_MS="${BE_ALERT_MS:-50}"
GW_ALERT_MS="${GW_ALERT_MS:-50}"
ROUTER_ALERT_MS="${ROUTER_ALERT_MS:-50}"
LOSS_ALERT_PCT="${LOSS_ALERT_PCT:-1}"
MDEV_ALERT_MS="${MDEV_ALERT_MS:-80}"
CONNTRACK_ALERT_PCT="${CONNTRACK_ALERT_PCT:-70}"
# ---------------------------------------------------------------------------
emit() {
_kind="$1"; _ts="$2"; _role="$3"; _host="$4"
shift 4
_line="kind=$_kind ts=$_ts ver=3 role=$_role host=$_host $*"
_dir="$(dirname "$WAN_WATCH_LOG")"
mkdir -p "$_dir" 2>/dev/null || true
printf '%s\n' "$_line" >> "$WAN_WATCH_LOG"
}
detect_role() {
_role="$WAN_WATCH_ROLE"
if [ "$_role" != "auto" ]; then printf '%s' "$_role"; return; fi
_hn="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)"
case "$_hn" in
*raspberry*|*rpi*) printf 'pi' ;;
*monstro*|*router*|*gw*) printf 'router' ;;
*gentoo*|*wg0*|*fe*) printf 'fe' ;;
*) printf 'host' ;;
esac
}
default_gw() { ip route show default 2>/dev/null | awk '{print $3; exit}'; }
route_dev() {
ip route get "$1" 2>/dev/null | awk '{for (i=1;i<=NF;i++) if ($i=="dev") {print $(i+1); exit}}'
}
route_via() {
ip route get "$1" 2>/dev/null | awk '{for (i=1;i<=NF;i++) if ($i=="via") {print $(i+1); exit}}'
}
load_1m() { awk '{print $1}' /proc/loadavg 2>/dev/null || echo n/a; }
cpu_busy_pct() {
awk '/^cpu /{idle=$5+$6;t=0;for(i=2;i<=NF;i++)t+=$i;if(t>0)printf"%.0f",100*(t-idle)/t}' /proc/stat 2>/dev/null || echo n/a
}
conntrack_kv() {
_c="$(cat /proc/sys/net/netfilter/nf_conntrack_count 2>/dev/null)" || _c=""
_m="$(cat /proc/sys/net/netfilter/nf_conntrack_max 2>/dev/null)" || _m=""
if [ -n "$_c" ] && [ -n "$_m" ] && [ "$_m" -gt 0 ] 2>/dev/null; then
printf 'conntrack=%s/%s(%s%%)' "$_c" "$_m" "$((_c * 100 / _m))"
fi
}
neigh_state() { ip neigh show "$1" 2>/dev/null | awk '{print $6; exit}'; }
num_ge() {
awk -v a="$1" -v b="$2" 'BEGIN{exit(a+0>=b+0)?0:1}' 2>/dev/null
}
ping_stats() {
_host="$1"; _cnt="${2:-$PING_COUNT}"; _dead="${3:-$PING_DEADLINE_S}"
_out="$(ping -c "$_cnt" -w "$_dead" "$_host" 2>/dev/null)" || _out=""
if [ -z "$_out" ]; then printf '%s %s %s %s %s' -1 -1 -1 -1 -1; return; fi
_loss="$(printf '%s\n' "$_out" | awk '/packet loss/{for(i=1;i<=NF;i++) if($i~/%$/){gsub(/%/,"",$i);print $i;exit}}')"
_rtt="$(printf '%s\n' "$_out" | sed -n 's/.*= \([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\)\/\([0-9.]*\) ms.*/\1 \2 \3 \4/p' | head -1)"
if [ -n "$_rtt" ]; then
set -- $_rtt
printf '%s %s %s %s %s' "${_loss:-0}" "$1" "$2" "$3" "$4"
else
_avg="$(printf '%s\n' "$_out" | awk -F'[=/ ]' '/min\/avg\/max/{print $8;exit}')"
printf '%s %s %s %s %s' "${_loss:-0}" "${_avg:--1}" "${_avg:--1}" "${_avg:--1}" 0
fi
}
append_ping() {
_line="$1"; _pfx="$2"; _host="$3"
set -- $(ping_stats "$_host")
_loss="$1"; _min="$2"; _avg="$3"; _max="$4"; _mdev="$5"
printf '%s %s=%s %s_loss=%s%% %s_min_ms=%s %s_avg_ms=%s %s_max_ms=%s %s_mdev_ms=%s' \
"$_line" "$_pfx" "$_host" "$_pfx" "$_loss" "$_pfx" "$_min" "$_pfx" "$_avg" "$_pfx" "$_max" "$_pfx" "$_mdev"
}
field_val() {
printf '%s\n' "$2" | awk -v f="$1" '{for(i=1;i<=NF;i++) if($i~"^"f"="){split($i,a,"=");gsub(/%/,"",a[2]);print a[2];exit}}'
}
acquire_lock() {
command -v flock >/dev/null 2>&1 || return 0
_lock="$WAN_WATCH_LOCK"
_fb="${TMPDIR:-/tmp}/wan-watch.$$.lock"
_dir="$(dirname "$_lock")"
if [ ! -w "$_dir" ] 2>/dev/null; then _lock="${TMPDIR:-/tmp}/wan-watch.$(id -u 2>/dev/null || echo 0).lock"; fi
exec 9>"$_lock" 2>/dev/null || return 0
flock -n 9 || { echo "wan-watch: skip (running)" >&2; exit 0; }
}
cooldown_active() {
[ -r "$WAN_WATCH_COOLDOWN_FILE" ] || return 1
_l="$(cat "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null)" || return 1
_n="$(date +%s 2>/dev/null)" || return 1
[ "$((_n - _l))" -lt "$WAN_WATCH_COOLDOWN_S" ] 2>/dev/null
}
# ---------------------------------------------------------------------------
# Probes: tunnels, services, kernel — cheap, every run
# ---------------------------------------------------------------------------
probe_emit() {
_ts="$1"; _role="$2"; _host="$3"; _section="$4"
shift 4
emit probe "$_ts" "$_role" "$_host" "section=$_section $*"
}
probe_all() {
_ts="$1"; _role="$2"; _host="$3"
# Kernel modules (GRE/PPTP)
if [ -r /proc/modules ]; then
_gre="$(grep -E '^ip_gre |^gre ' /proc/modules 2>/dev/null | awk '{print $1}' | tr '\n' ',' | sed 's/,$//')"
_pptp="$(grep -E 'nf_conntrack_pptp|nf_nat_pptp' /proc/modules 2>/dev/null | awk '{print $1}' | tr '\n' ',' | sed 's/,$//')"
[ -n "$_gre" ] && probe_emit "$_ts" "$_role" "$_host" gre "loaded=${_gre}"
[ -n "$_pptp" ] && probe_emit "$_ts" "$_role" "$_host" pptp_mod "loaded=${_pptp}"
fi
# GRE / gretap links
ip -d link show type gre 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" '
/^[0-9]+:/{name=$2; gsub(":","",name); state=$0; getline; detail=$0;
printf "kind=probe ts=%s ver=3 role=%s host=%s section=gre_iface name=%s up=%s\n", ts,role,host,name,(index(state,"UP")?"yes":"no")}
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
# PPTP listeners / sessions
_p1723="$(ss -lnpt 2>/dev/null | grep -c ':1723 ' || echo 0)"
_pppd="$(pgrep -c pppd 2>/dev/null || echo 0)"
_pptpd="$(pgrep -c pptpd 2>/dev/null || echo 0)"
probe_emit "$_ts" "$_role" "$_host" pptp "listen1723=${_p1723} pppd=${_pppd} pptpd=${_pptpd}"
# tinc
_tinc="$(pgrep -c tincd 2>/dev/null || echo 0)"
if [ "$_tinc" -gt 0 ] 2>/dev/null; then
_tn=""
if command -v tincctl >/dev/null 2>&1; then
for _net in /etc/tinc/*/tinc.conf; do
[ -f "$_net" ] || continue
_n="$(basename "$(dirname "$_net")")"
_st="$(tincctl -n "$_n" status 2>/dev/null | tr '\n' ' ' | sed 's/ */ /g')"
probe_emit "$_ts" "$_role" "$_host" tinc "net=${_n} tincd=${_tinc} status=${_st:-unknown}"
done
else
probe_emit "$_ts" "$_role" "$_host" tinc "tincd=${_tinc} status=no_tincctl"
fi
else
probe_emit "$_ts" "$_role" "$_host" tinc "tincd=0"
fi
# WireGuard
if command -v wg >/dev/null 2>&1; then
wg show all 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" '
/^interface:/ { iface=$2; peer=""; hs=""; ep=""; tx=""; rx="" }
/^ public key:/ { pk=$3 }
/^ endpoint:/ { ep=$2 }
/^ latest handshake:/ { hs=$0; sub(/^ latest handshake: /,"",hs) }
/^ transfer:/ { tx=$2; rx=$4 }
/^$/ {
if (iface != "")
printf "kind=probe ts=%s ver=3 role=%s host=%s section=wireguard iface=%s endpoint=%s handshake=%s tx=%s rx=%s\n",
ts, role, host, iface, ep, hs, tx, rx
}
END {
if (iface != "")
printf "kind=probe ts=%s ver=3 role=%s host=%s section=wireguard iface=%s endpoint=%s handshake=%s tx=%s rx=%s\n",
ts, role, host, iface, ep, hs, tx, rx
}
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
fi
# iodine / dns tunnel iface
_iod="$(pgrep -c iodined 2>/dev/null || echo 0)"
probe_emit "$_ts" "$_role" "$_host" iodine "iodined=${_iod}"
for _d in dns0 tun0 tap0; do
ip link show "$_d" >/dev/null 2>&1 || continue
ip -s link show "$_d" 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v d="$_d" '
/RX:/{getline; re=$3; rd=$4} /TX:/{getline; te=$3; td=$4}
END{printf "kind=probe ts=%s ver=3 role=%s host=%s section=tunnel_iface dev=%s rx_err=%s rx_drop=%s tx_err=%s tx_drop=%s\n",
ts,role,host,d,re+0,rd+0,te+0,td+0}
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
done
# Named / squid / nginx (edge services)
for _svc in named bind9 squid nginx iodined pptpd; do
_c="$(pgrep -c "$_svc" 2>/dev/null || echo 0)"
[ "$_c" -gt 0 ] 2>/dev/null || continue
_cpu="$(ps -eo pcpu,comm 2>/dev/null | awk -v s="$_svc" '$2~s{print $1; exit}')"
probe_emit "$_ts" "$_role" "$_host" service "name=${_svc} procs=${_c} cpu=${_cpu:-n/a}"
done
# Key ifaces stats
for _d in xenbr0 eno0 eth0 wlan0 br0; do
ip link show "$_d" >/dev/null 2>&1 || continue
ip -s link show "$_d" 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v d="$_d" '
/RX:/{getline; re=$3; rd=$4} /TX:/{getline; te=$3; td=$4}
END{printf "kind=probe ts=%s ver=3 role=%s host=%s section=iface dev=%s rx_drop=%s tx_drop=%s rx_err=%s tx_err=%s\n",
ts,role,host,d,rd+0,td+0,re+0,te+0}
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
tc -s qdisc show dev "$_d" 2>/dev/null | head -3 | while IFS= read -r _ql; do
probe_emit "$_ts" "$_role" "$_host" qdisc "dev=${_d} detail=${_ql}"
done
done
# softnet drops
if [ -r /proc/net/softnet_stat ]; then
awk -v ts="$_ts" -v role="$_role" -v host="$_host" '$2+0>0{
printf "kind=probe ts=%s ver=3 role=%s host=%s section=softnet cpu=%d dropped=%s\n",ts,role,host,NR-1,$2
}' /proc/net/softnet_stat >> "$WAN_WATCH_LOG" 2>/dev/null || true
fi
# Routes + neigh for key targets
for _label in wan:$WAN_PING_TARGET fe:$FE_PING_TARGET gw:${GW_PING_TARGET:-auto}; do
_l="${_label%%:*}"; _t="${_label#*:}"
[ "$_t" = "auto" ] && _t="$(default_gw)"
[ -n "$_t" ] && [ "$_t" != none ] || continue
ip route get "$_t" 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v l="$_l" -v t="$_t" '
{dev=via=src=""; for(i=1;i<=NF;i++){if($i=="dev")dev=$(i+1);if($i=="via")via=$(i+1);if($i=="src")src=$(i+1)}
printf "kind=probe ts=%s ver=3 role=%s host=%s section=route label=%s target=%s dev=%s via=%s src=%s\n",ts,role,host,l,t,dev,via,src}
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
done
_ns="$(neigh_state "$FE_PING_TARGET")"
[ -n "$_ns" ] && probe_emit "$_ts" "$_role" "$_host" neigh "ip=${FE_PING_TARGET} state=${_ns}"
# iptables hot rules (router/fe)
if command -v iptables >/dev/null 2>&1 && [ "$_role" = "router" ]; then
for _spec in filter:FORWARD nat:PREROUTING nat:POSTROUTING; do
_tbl="${_spec%%:*}"; _ch="${_spec#*:}"
iptables -t "$_tbl" -L "$_ch" -n -v --line-numbers 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" -v tbl="$_tbl" -v ch="$_ch" '
/^[0-9]+/ && $2+0>1000 {
printf "kind=probe ts=%s ver=3 role=%s host=%s section=iptables tbl=%s chain=%s line=%s pkts=%s target=%s\n",
ts,role,host,tbl,ch,$1,$2,$4
}' | head -6 >> "$WAN_WATCH_LOG" 2>/dev/null || true
done
fi
# Top CPU
ps -eo pcpu=,pmem=,comm= --sort=-pcpu 2>/dev/null | head -5 | awk -v ts="$_ts" -v role="$_role" -v host="$_host" '
NF>=3{printf "kind=probe ts=%s ver=3 role=%s host=%s section=proc rank=%d cpu=%s mem=%s comm=%s\n",ts,role,host,NR,$1,$2,$3}
' >> "$WAN_WATCH_LOG" 2>/dev/null || true
ss -s 2>/dev/null | awk -v ts="$_ts" -v role="$_role" -v host="$_host" '/TCP:|UDP:/{
gsub(/^[ \t]+/,""); printf "kind=probe ts=%s ver=3 role=%s host=%s section=ss %s\n",ts,role,host,$0
}' >> "$WAN_WATCH_LOG" 2>/dev/null || true
}
# ---------------------------------------------------------------------------
# Deep capture (on anomaly)
# ---------------------------------------------------------------------------
deep_burst_line() {
_ts="$1"; _role="$2"; _host="$3"; _label="$4"; _target="$5"
set -- $(ping_stats "$_target" "$DEEP_PING_COUNT" "$DEEP_PING_DEADLINE_S")
emit deep "$_ts" "$_role" "$_host" \
"parent_ts=$_ts section=burst label=$_label target=$_target loss=$1% min_ms=$2 avg_ms=$3 max_ms=$4 mdev_ms=$5"
}
deep_capture() {
_ts="$1"; _role="$2"; _host="$3"; _triggers="$4"
emit alert "$_ts" "$_role" "$_host" "parent_ts=$_ts triggers=\"$_triggers\""
_ct="$(conntrack_kv)"
emit deep "$_ts" "$_role" "$_host" "parent_ts=$_ts section=host load1=$(load_1m) cpu_busy=$(cpu_busy_pct)% mem_avail_kb=$(awk '/^MemAvailable:/{print $2}' /proc/meminfo 2>/dev/null || echo n/a) ${_ct}"
deep_burst_line "$_ts" "$_role" "$_host" wan "$WAN_PING_TARGET"
deep_burst_line "$_ts" "$_role" "$_host" fe "$FE_PING_TARGET"
_gw="$GW_PING_TARGET"; [ "$_gw" = auto ] && _gw="$(default_gw)"
[ -n "$_gw" ] && [ "$_gw" != none ] && deep_burst_line "$_ts" "$_role" "$_host" gw "$_gw"
probe_all "$_ts" "$_role" "$_host"
}
detect_triggers() {
_line="$1"; _role="$2"; _t=""
for _p in wan fe be gw router; do
_a="$(field_val "${_p}_avg_ms" "$_line")"
_l="$(field_val "${_p}_loss" "$_line")"
_m="$(field_val "${_p}_mdev_ms" "$_line")"
_thr="$FE_ALERT_MS"
case "$_p" in wan) _thr="$WAN_ALERT_MS";; fe) _thr="$FE_ALERT_MS";;
be) _thr="$BE_ALERT_MS";; gw) _thr="$GW_ALERT_MS";;
router) _thr="$ROUTER_ALERT_MS";; esac
num_ge "$_a" "$_thr" && _t="${_t}${_t:+,}${_p}_avg=${_a}"
num_ge "$_l" "$LOSS_ALERT_PCT" && _t="${_t}${_t:+,}${_p}_loss=${_l}"
num_ge "$_m" "$MDEV_ALERT_MS" && _t="${_t}${_t:+,}${_p}_mdev=${_m}"
done
_ct="$(printf '%s\n' "$_line" | sed -n 's/.*(\([0-9]*\)%).*/\1/p' | head -1)"
num_ge "$_ct" "$CONNTRACK_ALERT_PCT" && _t="${_t}${_t:+,}conntrack_pct=${_ct}"
_wan="$(field_val wan_avg_ms "$_line")"; _fe="$(field_val fe_avg_ms "$_line")"
if [ "$_role" = pi ] && num_ge "$_fe" "$FE_ALERT_MS" && ! num_ge "$_wan" "$WAN_ALERT_MS"; then
case "$_t" in *fe_avg=*) ;; *) _t="${_t}${_t:+,}pi_fe_path";; esac
fi
if [ "$_role" = router ] && num_ge "$_wan" "$WAN_ALERT_MS" && ! num_ge "$_fe" "$FE_ALERT_MS"; then
case "$_t" in *wan_avg=*) ;; *) _t="${_t}${_t:+,}router_wan_path";; esac
fi
printf '%s' "$_t"
}
# ---------------------------------------------------------------------------
# Verdict engine — runs EVERY time on current snapshot
# ---------------------------------------------------------------------------
compute_verdict() {
_line="$1"; _role="$2"
_probe_hint="${3:-}"
_wan="$(field_val wan_avg_ms "$_line")"
_fe="$(field_val fe_avg_ms "$_line")"
_be="$(field_val be_avg_ms "$_line")"
_gw="$(field_val gw_avg_ms "$_line")"
_wmdev="$(field_val wan_mdev_ms "$_line")"
_ct="$(printf '%s\n' "$_line" | sed -n 's/.*(\([0-9]*\)%).*/\1/p' | head -1)"
_verdict="OK"
_conf="high"
_summary="All probes within thresholds."
_action="none"
if [ "$_role" = router ]; then
if num_ge "$_wan" "$WAN_ALERT_MS" && ! num_ge "$_fe" 5; then
_verdict="MONSTRO_WAN_BLOAT"
_summary="WAN leg slow (~${_wan}ms) but FE local OK (~${_fe}ms). Edge xenbr0/upstream queue."
_action="SQM/fq_codel on xenbr0; check gw $(route_via "$WAN_PING_TARGET" 2>/dev/null || echo 10.7.16.228); review squid/named load"
elif num_ge "$_gw" "$GW_ALERT_MS" && num_ge "$_wan" "$WAN_ALERT_MS"; then
_verdict="UPSTREAM_GW"
_summary="WAN (${_wan}ms) and gw (${_gw}ms) both high — parent hop 10.7.16.228 stressed."
_action="Check upstream link; correlate with ISP/gateway admin"
elif num_ge "$_ct" "$CONNTRACK_ALERT_PCT"; then
_verdict="CONNTRACK_PRESSURE"
_summary="Conntrack at ${_ct}% — new flows may stall."
_action="Inspect conntrack count; reduce NAT/tunnel sessions or raise nf_conntrack_max"
elif num_ge "$_wmdev" "$MDEV_ALERT_MS"; then
_verdict="WAN_JITTER"
_summary="WAN jitter high (mdev=${_wmdev}ms) avg=${_wan}ms."
_action="Bufferbloat likely; enable fq_codel"
fi
elif [ "$_role" = pi ]; then
if num_ge "$_fe" "$FE_ALERT_MS" && ! num_ge "$_wan" "$WAN_ALERT_MS"; then
_verdict="PATH_TO_FE"
_summary="Pi ISP OK (~${_wan}ms) but FE path slow (~${_fe}ms). Problem at monstro edge, not Pi WiFi."
_action="Check monstro wan-watch verdict; your traffic crosses modem WAN queue"
elif num_ge "$_wan" "$WAN_ALERT_MS"; then
_verdict="PI_ISP"
_summary="Pi WAN to 1.1.1.1 slow (${_wan}ms)."
_action="Local ISP/WiFi issue on Pi side"
fi
elif [ "$_role" = fe ]; then
if num_ge "$_wan" "$WAN_ALERT_MS"; then
_verdict="FE_WAN_EGRESS"
_summary="FE WAN egress slow (${_wan}ms)."
_action="Check FE default route and nginx upstream separately"
fi
fi
# Probe hints from same run (grep log tail — set by caller in _probe_hint)
case "${_probe_hint:-}" in
*dns0*tx_drop=*|*tunnel_iface*dns0*)
if [ "$_verdict" = "OK" ] || [ "$_verdict" = "MONSTRO_WAN_BLOAT" ]; then
_verdict="TUNNEL_DNS0_LOAD"
_summary="${_summary} iodine/dns0 tunnel drops elevated."
_action="${_action}; inspect iodined and dns0 tx_drop"
_conf="medium"
fi ;;
*named*cpu=*)
if [ "$_verdict" = "MONSTRO_WAN_BLOAT" ] || [ "$_verdict" = "OK" ]; then
_summary="${_summary} named DNS active on router."
_conf="medium"
fi ;;
esac
printf 'verdict=%s confidence=%s summary="%s" action="%s" wan_ms=%s fe_ms=%s gw_ms=%s' \
"$_verdict" "$_conf" "$_summary" "$_action" "$_wan" "$_fe" "$_gw"
}
# ---------------------------------------------------------------------------
# Main cron run
# ---------------------------------------------------------------------------
run_watch() {
command -v ping >/dev/null 2>&1 || { echo "wan-watch: need ping" >&2; exit 1; }
acquire_lock
_ts="$(date -Iseconds 2>/dev/null || date '+%Y-%m-%dT%H:%M:%S%z')"
_role="$(detect_role)"
_host="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)"
_gw="$GW_PING_TARGET"; [ "$_gw" = auto ] && _gw="$(default_gw)"
_ct="$(conntrack_kv)"
_wdev="$(route_dev "$WAN_PING_TARGET")"; _fdev="$(route_dev "$FE_PING_TARGET")"
_wgw="$(route_via "$WAN_PING_TARGET")"
_ns="$(neigh_state "$FE_PING_TARGET")"
_snap="$(append_ping "" wan "$WAN_PING_TARGET")"
_snap="$(append_ping "$_snap" fe "$FE_PING_TARGET")"
[ -n "$BE_PING_TARGET" ] && [ "$BE_PING_TARGET" != none ] && _snap="$(append_ping "$_snap" be "$BE_PING_TARGET")"
[ -n "$ROUTER_PING_TARGET" ] && [ "$ROUTER_PING_TARGET" != none ] && _snap="$(append_ping "$_snap" router "$ROUTER_PING_TARGET")"
[ -n "$_gw" ] && [ "$_gw" != none ] && _snap="$(append_ping "$_snap" gw "$_gw")"
_snap="$_snap load1=$(load_1m) cpu_busy=$(cpu_busy_pct)%"
[ -n "$_ct" ] && _snap="$_snap $_ct"
[ -n "$_wdev" ] && _snap="$_snap route_wan=$_wdev"
[ -n "$_fdev" ] && _snap="$_snap route_fe=$_fdev"
[ -n "$_wgw" ] && _snap="$_snap via_wan=$_wgw"
[ -n "$_ns" ] && _snap="$_snap neigh_fe=$_ns"
emit snapshot "$_ts" "$_role" "$_host" "$_snap" >/dev/null
probe_all "$_ts" "$_role" "$_host"
_probe_hint="$(tail -30 "$WAN_WATCH_LOG" 2>/dev/null | grep 'kind=probe' | tr '\n' ' ')"
_verdict_kv="$(compute_verdict "$_snap" "$_role" "$_probe_hint")"
_verdict_line="kind=verdict ts=$_ts ver=3 role=$_role host=$_host $_verdict_kv"
printf '%s\n' "$_verdict_line" >> "$WAN_WATCH_LOG"
_triggers="$(detect_triggers "$_snap" "$_role")"
_do_deep=0
if [ -n "$_triggers" ]; then _do_deep=1; fi
if [ "$WAN_WATCH_DEEP" = "always" ]; then _do_deep=1; fi
if [ "$WAN_WATCH_DEEP" = "never" ]; then _do_deep=0; fi
if [ "$_do_deep" = 1 ]; then
if cooldown_active && [ "$WAN_WATCH_DEEP" != "always" ]; then
emit alert "$_ts" "$_role" "$_host" "parent_ts=$_ts triggers=\"$_triggers\" action=skipped_cooldown" >/dev/null
else
deep_capture "$_ts" "$_role" "$_host" "$_triggers"
date +%s > "$WAN_WATCH_COOLDOWN_FILE" 2>/dev/null || true
fi
fi
# Human one-liner to cron log
printf 'WAN-WATCH [%s/%s] %s\n' "$_role" "$_host" "$_verdict_kv"
}
# ---------------------------------------------------------------------------
# --analyze: final conclusion from log without waiting
# ---------------------------------------------------------------------------
analyze_log() {
_log="$WAN_WATCH_LOG"
_hours="${1:-$WAN_ANALYZE_HOURS}"
[ -r "$_log" ] || { echo "wan-watch: no log $_log" >&2; exit 1; }
_cut="$(date -d "${_hours} hours ago" '+%Y-%m-%dT%H:%M' 2>/dev/null || echo "")"
echo "=== wan-watch analyze (last ${_hours}h) ==="
echo "log=$_log"
echo ""
awk -v cut="$_cut" '
function ts_ok(t) { return (cut == "" || t >= cut) }
/^kind=snapshot / && ts_ok($0) {
for(i=1;i<=NF;i++) {
if($i~/^wan_avg_ms=/){split($i,a,"="); wa=a[2]+0; wn++}
if($i~/^fe_avg_ms=/){split($i,a,"="); fa=a[2]+0; fn++}
if($i~/^gw_avg_ms=/){split($i,a,"="); ga=a[2]+0; gn++}
if($i~/^role=/){split($i,a,"="); role=a[2]}
}
if(wa>=50){wb++} else if(wa>=0){wg++}
if(fa>=50){fb++} else if(fa>=0){fg++}
if(wa>=50 && fa<5) rw_ok_fe++ # router pattern
if(fa>=50 && wa<30) pi_bad++ # pi pattern
sn++
}
/^kind=verdict / && ts_ok($0) {
if($0~/verdict=OK/) okv++; else badv++
for(i=1;i<=NF;i++) if($i~/^verdict=/){split($i,a,"="); vc[a[2]]++}
}
/^kind=probe / && /section=tunnel_iface/ && /tx_drop=[1-9]/ { dns_drop++ }
/^kind=probe / && /section=service/ && /name=named/ { named++ }
/^kind=probe / && /section=service/ && /name=squid/ { squid++ }
END {
printf "snapshots=%d (wan_bad=%d wan_good=%d fe_bad=%d fe_good=%d)\n", sn, wb+0, wg+0, fb+0, fg+0
printf "router_wan_bad_fe_local_ok=%d (MONSTRO_WAN_BLOAT signature)\n", rw_ok_fe+0
printf "pi_fe_bad_wan_ok=%d (PATH_TO_FE signature)\n", pi_bad+0
printf "verdicts: ok=%d anomaly=%d\n", okv+0, badv+0
printf "verdict_counts:"
for(v in vc) printf " %s=%d", v, vc[v]
printf "\n"
printf "probe_hints: dns0_tx_drop_events=%d named_samples=%d squid_samples=%d\n", dns_drop+0, named+0, squid+0
pct = (sn>0) ? int(100*(wb+fb)/(2*sn)) : 0
printf "\n"
if(rw_ok_fe>=3 || pi_bad>=3) {
printf "CONCLUSION: MONSTRO WAN EDGE QUEUEING (confidence=HIGH)\n"
printf " Evidence: wan slow + fe local OK; Pi ISP stays ~9ms in prior data.\n"
printf " Not: FE VM, BE, Pi WiFi, global ISP.\n"
printf " Fix order: 1) fq_codel/SQM on xenbr0 2) gw 10.7.16.228 3) squid/named 4) iodine dns0\n"
} else if(wb+fb==0) {
printf "CONCLUSION: HEALTHY (confidence=HIGH)\n"
} else {
printf "CONCLUSION: MIXED/INTERMITTENT (%d%% bad samples) — keep */5 cron running\n", pct
}
}
' "$_log"
echo ""
echo "=== last 5 verdicts ==="
grep '^kind=verdict ' "$_log" 2>/dev/null | tail -5 | while IFS= read -r _l; do
_t="$(printf '%s\n' "$_l" | sed 's/.*ts=\([^ ]*\).*/\1/')"
_v="$(printf '%s\n' "$_l" | sed 's/.*verdict=\([^ ]*\).*/\1/')"
_s="$(printf '%s\n' "$_l" | sed 's/.*summary="\([^"]*\)".*/\1/')"
printf ' %s %s %s\n' "$_t" "$_v" "$_s"
done
}
timeline_log() {
_log="$WAN_WATCH_LOG"
[ -r "$_log" ] || { echo "no log"; exit 1; }
echo "=== hourly (snapshot wan/fe bad if avg>=50) ==="
awk '
/^kind=snapshot / {
ts=$0; sub(/.*ts=/,"",ts); sub(/ .*/,"",ts)
hour=substr(ts,1,13)
for(i=1;i<=NF;i++){
if($i~/^wan_avg_ms=/){split($i,a,"="); w=a[2]+0}
if($i~/^fe_avg_ms=/){split($i,a,"="); f=a[2]+0}
}
t[hour]++; if(w>=50||f>=50) b[hour]++
}
END {
for(h in t){ bh=b[h]+0; printf " %s %d/%d bad ", h, bh, t[h]
for(i=0;i<t[h];i++) printf(i<bh?"#":".")
printf "\n"
}
}
' "$_log" | sort
}
full_report() {
_ts="$(date -Iseconds 2>/dev/null || date '+%Y-%m-%dT%H:%M:%S%z')"
_role="$(detect_role)"
_host="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo unknown)"
echo "=== wan-watch FULL @ $_ts role=$_role host=$_host ==="
PING_COUNT=5 PING_DEADLINE_S=10
for _t in "$WAN_PING_TARGET" "$FE_PING_TARGET" "$BE_PING_TARGET" "$ROUTER_PING_TARGET" "$(default_gw)"; do
[ -n "$_t" ] && [ "$_t" != none ] || continue
echo "--- ping $_t ---"
ping -c 5 -w 10 "$_t" 2>/dev/null || echo "fail"
done
echo "--- host ---"
echo "load=$(load_1m) cpu=$(cpu_busy_pct)% $(conntrack_kv)"
echo "--- probes ---"
probe_all "$_ts" "$_role" "$_host"
tail -40 "$WAN_WATCH_LOG" | grep "^kind=probe "
echo "--- verdict ---"
_snap="ts=$_ts role=$_role host=$_host"
_snap="$(append_ping "$_snap" wan "$WAN_PING_TARGET")"
_snap="$(append_ping "$_snap" fe "$FE_PING_TARGET")"
compute_verdict "$_snap" "$_role"
echo ""
}
correlate_logs() {
if [ "$#" -lt 1 ]; then echo "wan-watch: --correlate needs log files" >&2; exit 1; fi
_tmp="$(mktemp "${TMPDIR:-/tmp}/wan-corr.XXXXXX")"
trap 'rm -f "$_tmp"' EXIT INT HUP
for _path in "$@"; do
[ -r "$_path" ] || continue
while IFS= read -r _line || [ -n "$_line" ]; do
case "$_line" in
kind=snapshot*) _ts="${_line#*ts=}"; _ts="${_ts%% *}" ;;
ts=*) _ts="${_line#ts=}"; _ts="${_ts%% *}" ;;
*) continue ;;
esac
_date="${_ts%%T*}"; _rest="${_ts#*T}"; _hour="${_rest%%:*}"
_min="${_rest#*:}"; _min="${_min%%:*}"; _min="${_min%%+*}"; _min="${_min%%-*}"
_slot="${_date}T${_hour}:$(printf '%02d' $((_min / 15 * 15)))"
_role="$(printf '%s\n' "$_line" | awk '{for(i=1;i<=NF;i++) if($i~/^role=/){split($i,a,"=");print a[2];exit}}')"
[ -n "$_role" ] || _role="$(basename "$_path" .log)"
printf '%s\t%s\t%s\t%s\n' "$_slot" "$_role" "$_ts" "$_line" >> "$_tmp"
done < "$_path"
done
printf 'slot\t'
_roles="$(awk -F'\t' '{print $2}' "$_tmp" | sort -u | tr '\n' ' ')"
for _r in $_roles; do printf '%s_wan\t%s_fe\t%s_verdict\t' "$_r" "$_r" "$_r"; done
printf '\n'
awk -F'\t' '
{
slot=$1; role=$2; ts=$3; line=$4; key=slot SUBSEP role
if (!(key in best) || ts>best[key]) {
best[key]=ts; wan=fe="-"; ver="-"
for(i=1;i<=split(line,a," ");i++){
if(a[i]~/^wan_avg_ms=/){split(a[i],b,"=");wan=b[2]}
if(a[i]~/^fe_avg_ms=/){split(a[i],b,"=");fe=b[2]}
}
W[slot,role,"wan"]=wan; W[slot,role,"fe"]=fe
slots[slot]=1; roles[role]=1
}
}
END{
nr=0; for(r in roles) rl[++nr]=r; asort(rl)
ns=0; for(s in slots) sl[++ns]=s; asort(sl)
for(i=1;i<=ns;i++){
s=sl[i]; printf "%s", s
for(j=1;j<=nr;j++){
r=rl[j]; printf "\t%s\t%s\t-", W[s,r,"wan"]+0?W[s,r,"wan"]:"-", W[s,r,"fe"]+0?W[s,r,"fe"]:"-"
}
printf "\n"
}
}
' "$_tmp"
}
usage() {
cat <<EOF
wan-watch.sh — unified network watchdog (ver=3)
Cron: */5 * * * * root . /etc/conf.d/wan-watch; /usr/local/bin/wan-watch.sh
Commands:
(default) snapshot + probes + verdict (+ deep if bad)
--full immediate human report
--analyze [H] conclusion from log (default ${WAN_ANALYZE_HOURS}h)
--timeline hourly bad/good bars
--force-deep set WAN_WATCH_DEEP=always for one run
--tail [N] last N snapshot+verdict lines
--correlate FILE… merge Pi+router logs by 15-min slot
--help
Log: $WAN_WATCH_LOG (kind=snapshot|probe|verdict|alert|deep)
EOF
}
case "${1:-}" in
-h|--help|help) usage ;;
--full|full) full_report ;;
--analyze|analyze) analyze_log "${2:-$WAN_ANALYZE_HOURS}" ;;
--timeline|timeline) timeline_log ;;
--force-deep|force-deep) WAN_WATCH_DEEP=always run_watch ;;
--tail|tail)
_n="${2:-20}"
grep -E '^kind=(snapshot|verdict|alert) ' "$WAN_WATCH_LOG" 2>/dev/null | tail -n "$_n" || true
;;
"") run_watch ;;
*) echo "wan-watch: unknown $1"; usage; exit 1 ;;
esac

2
examples/wireguard/alpine-be-sudoers-wg
View File

@@ -4,3 +4,5 @@ Defaults:nobody !requiretty
Defaults:nginx !requiretty Defaults:nginx !requiretty
nobody ALL=(root) NOPASSWD: /usr/bin/wg set wg0 peer * nobody ALL=(root) NOPASSWD: /usr/bin/wg set wg0 peer *
nginx ALL=(root) NOPASSWD: /usr/bin/wg set wg0 peer * nginx ALL=(root) NOPASSWD: /usr/bin/wg set wg0 peer *
nginx ALL=(root) NOPASSWD: /usr/bin/wg show wg0 dump
nobody ALL=(root) NOPASSWD: /usr/bin/wg show wg0 dump

2
examples/wireguard/install_be_remote_access.sh
View File

@@ -28,7 +28,7 @@ $c["remote_access"] = array_merge($c["remote_access"] ?? [], [
"wg_client_ip_prefix" => "172.200.2.", "wg_client_ip_prefix" => "172.200.2.",
"wg_client_ip_min" => 10, "wg_client_ip_min" => 10,
"wg_client_ip_max" => 250, "wg_client_ip_max" => 250,
"wg_set_prefix" => "sudo ", "wg_set_prefix" => "sudo -n ",
"provision_peers" => true, "provision_peers" => true,
"require_wg_tools" => true, "require_wg_tools" => true,
"session_ttl_s" => 3600, "session_ttl_s" => 3600,

77
orchestration/sim/cluster0/ARCHITECTURE.md Normal file
View File

@@ -0,0 +1,77 @@
# cast01–cast03 lab architecture
## Goals
Simulate production patterns on three isolated Alpine VMs with **shared config only** on NFS (`/shared/cluster/`). No database files on NFS — each node owns `/var/lib/mysql` locally.
## Topology
```text
┌─────────────────────────────────────┐
│ FE 10.7.0.10 (NFS export /shared) │
└──────────────────┬──────────────────┘
│ NFS (configs/scripts only)
┌─────────────────────────────┼─────────────────────────────┐
▼ ▼ ▼
┌───────────┐ ┌───────────┐ ┌───────────┐
│ cast01 │ async GTID │ cast02 │ async GTID │ cast03 │
│ .16.236 │◄──────────────│ .16.237 │◄──────────────│ .16.238 │
│ PRIMARY │ replica │ REPLICA │ replica │ REPLICA │
│ MariaDB │ │ MariaDB │ │ MariaDB │
│ nginx/php │ │ nginx/php │ │ nginx/php │
└───────────┘ └───────────┘ └───────────┘
▲ │ │
└──────── writes + default reads ────────────────────────────┘
(replicas: read-only, failover drill)
```
| Node | IP | MariaDB | App role (planned) |
|------|-----|---------|-------------------|
| cast01 | 10.7.16.236 | **Primary** (read/write) | Crashes console primary target |
| cast02 | 10.7.16.237 | Replica (read_only) | App worker / read replica |
| cast03 | 10.7.16.238 | Replica (read_only) | App worker / read replica |
## MariaDB
- **Engine:** MariaDB 11.8 (Alpine 3.24 packages — no edge required)
- **Replication:** GTID async primary → two replicas (`server-id` 1/2/3)
- **Bootstrap:** app DB dump over TCP (`--skip-ssl`); GTID position copied from primary; `replicate-do-db` filters ongoing binlog to app schemas only (avoids `mysql.help_topic` dump failures)
- **Databases:** `androidcast_crashes`, `url_shortener` (schemas from project SQL in `/shared/cluster/sql/`)
- **Credentials:** see `credentials.txt` (same lab password on all nodes)
## PHP
- **Version:** 8.3 (`php83-fpm`) on all nodes — matches modern Alpine; prod BE remains 8.1 until upgraded separately
- **Socket:** `127.0.0.1:9000`
## DNS
Pending external records. Until then: `/etc/hosts` block maintained by `baseline.sh`.
## Service split (selected for lab)
| Service | Write DB host | Read DB host (lab default) | Future app path (local disk) |
|---------|---------------|----------------------------|------------------------------|
| Crashes / tickets / RA | cast01 | cast01 (cast02/03 for drill) | `/var/www/androidcast/` per node |
| URL shortener API | cast01 | cast01 | `/var/www/url-shortener/` on cast03 |
| Graphs ingest | cast01 | cast01 | shared crashes backend |
App deploys stay on **local VM disk**, not NFS.
## Operations
```sh
sudo sh /shared/cluster/scripts/baseline.sh # per-node baseline
sudo sh /shared/cluster/scripts/mariadb-primary.sh # cast01 only
sudo sh /shared/cluster/scripts/load-schemas.sh # cast01 only
sudo sh /shared/cluster/scripts/mariadb-replica.sh # cast02 + cast03
sudo sh /shared/cluster/scripts/verify-cluster.sh # any node
```
NFS mount may be `noexec` — always invoke scripts with `sh`.
## Not in scope yet
- ProxySQL / MaxScale (add if read splitting becomes necessary)
- Galera multi-primary (overkill for 3-node lab; async matches prod BE migration path)
- TLS between nodes (intra VLAN)

108
orchestration/sim/cluster0/DEPLOY.md Normal file
View File

@@ -0,0 +1,108 @@
# cast cluster — unattended populate & verify
This directory is the **single source of truth** for lab/staging cluster deployment.
Store scripts, configs, SQL, and credentials here only — **never** MariaDB datadirs or dumps.
## Fresh cluster workflow (cluster0 → clusterN)
### 0. FE: create NFS export + seed this tree
Copy or rsync this entire `cluster/` tree to the FE export, e.g.:
`10.7.0.10:/mnt/raid0/xendomains/domU_cast_cluster_0/shared/cluster/`
Edit `cluster.env` for the new cluster (IPs, export path, `CLUSTER_NAME`, `PUBLIC_ORIGIN`).
### 1. Create 3 empty Alpine VMs
Each VM needs hostname `cast01` / `cast02` / `cast03` and static IP from `cluster.env`.
Copy bootstrap env once (before NFS is mounted):
```sh
sudo install -m 644 bootstrap/cluster0-bootstrap.env /etc/cast-cluster.env
```
### 2. Populate (unattended)
**Option A — coordinator** (from a host with passwordless SSH to all cast nodes — typically your workstation, not cast01):
```sh
ssh ai@cast01 # or run from laptop with SSH config for cast01–03
sudo sh /shared/cluster/populate_lab_setup.sh --coordinator
```
If inter-node SSH is not configured, run **Option B** on each VM instead; cluster verify uses HTTP checks to peer IPs.
**Option B — per node** (parallel OK; replicas wait for primary):
```sh
sudo sh /shared/cluster/populate_lab_setup.sh
```
Phases:
| Phase | What |
|-------|------|
| `baseline` | NFS, hosts, `apk upgrade`, nginx/php83, packages |
| `db` | cast01: GTID primary + schemas; cast02/03: replicas |
| `app` | git/bundle deploy, nginx vhost, lab `config.php` |
| `all` | baseline → db → app → `verify_lab_setup.sh --local` |
Always invoke with `sh` — NFS may be mounted `noexec`.
### 3. DNS + FE redirect
Update external DNS and FE nginx upstream to point at the new cluster (`FE_PROXY_TARGET` in `cluster.env`).
### 4. Global verify
```sh
sudo sh /shared/cluster/verify_global_setup.sh
```
Checks `PUBLIC_ORIGIN` paths (crashes UI, assets, diag, short links).
## Staging / cloud
Same scripts; change only `cluster.env`:
- `FE_NFS_*` → object storage or config bucket mount
- `APP_SOURCE=bundle` + `release/current/` tarball from CI
- `PUBLIC_ORIGIN` → staging URL
- IPs / hostnames → cloud private network
## Files
| File | Purpose |
|------|---------|
| `cluster.env` | Cluster-specific variables (IPs, NFS, URLs, git) |
| `credentials.txt` | Shared lab passwords |
| `populate_lab_setup.sh` | Main unattended installer |
| `verify_lab_setup.sh` | Internal cluster health |
| `verify_global_setup.sh` | Post-DNS public checks |
| `scripts/` | Baseline, MariaDB, deploy, nginx |
| `sql/` | Schema + migrations |
| `nginx/apps-port80.conf` | php83-fpm TCP vhost |
| `bootstrap/` | Minimal `/etc/cast-cluster.env` for first boot |
See `ARCHITECTURE.md` for topology.
## Journal (read from FE if VMs fail)
Every populate/verify step appends to **`/shared/journal/{CLUSTER_NAME}.journal.tsv`** with status **OK**, **NOK**, **WARN**, or **START**.
Human summary (regenerated automatically):
```sh
less /shared/journal/cluster0.summary.md
grep NOK /shared/journal/cluster0.journal.tsv
```
On FE host:
```sh
less /mnt/raid0/xendomains/domU_cast_cluster_0/shared/journal/cluster0.summary.md
```
See `/shared/journal/README.md`. VM disk snapshots are out-of-band (ask sysop) — not stored on NFS.

17
orchestration/sim/cluster0/README.md Normal file
View File

@@ -0,0 +1,17 @@
# cast01–cast03 lab cluster
**Runbook:** [DEPLOY.md](DEPLOY.md) — unattended populate & verify.
Quick start on a fresh node:
```sh
sudo install -m 644 bootstrap/cluster0-bootstrap.env /etc/cast-cluster.env # once, before NFS
sudo sh /shared/cluster/populate_lab_setup.sh # or --coordinator
sudo sh /shared/cluster/verify_lab_setup.sh --local
# after DNS:
sudo sh /shared/cluster/verify_global_setup.sh
```
See `ARCHITECTURE.md` for topology and `cluster.env` for cluster-specific settings.
**Journal:** every setup action is logged under `/shared/journal/` with **OK** / **NOK** — readable from FE if VMs are down. See `/shared/journal/README.md`.

5
orchestration/sim/cluster0/bootstrap/cluster0-bootstrap.env Normal file
View File

@@ -0,0 +1,5 @@
# Copy to /etc/cast-cluster.env on each fresh VM before first populate run.
# Only needed to mount NFS when /shared/cluster/cluster.env is not reachable yet.
FE_NFS_SERVER=10.7.0.10
FE_NFS_EXPORT_PATH=/mnt/raid0/xendomains/domU_cast_cluster_0/shared
SHARED_MOUNT=/shared

42
orchestration/sim/cluster0/cluster.env Normal file
View File

@@ -0,0 +1,42 @@
# cluster0 — cast01–cast03 lab (edit for new clusters: cluster1, staging, etc.)
CLUSTER_NAME=cluster0
CLUSTER_ID=0
FE_NFS_SERVER=10.7.0.10
FE_NFS_EXPORT_PATH=/mnt/raid0/xendomains/domU_cast_cluster_0/shared
SHARED_MOUNT=/shared
CAST01_HOST=cast01
CAST01_IP=10.7.16.236
CAST02_HOST=cast02
CAST02_IP=10.7.16.237
CAST03_HOST=cast03
CAST03_IP=10.7.16.238
CAST_DOMAIN=intra.raptor.org
PRIMARY_DB_HOST=cast01
PRIMARY_DB_PORT=3306
ARTC0_HOST=artc0
ARTC0_IP=10.7.16.128
SSH_USER=ai
GIT_ORIGIN=git://f0xx.org/android_cast
GIT_BRANCH=next
# git | bundle — bundle uses SHARED_MOUNT/cluster/release/current/
APP_SOURCE=git
APP_ROOT=/var/www/localhost/htdocs/apps/app/androidcast_project
APP_BASE_PATH=/app/androidcast_project/crashes
APP_HUB_PATH=/app/androidcast_project
# Set after DNS + FE redirect, then run verify_global_setup.sh
PUBLIC_ORIGIN=https://apps.f0xx.org
SHORT_LINKS_PUBLIC_BASE=https://s.f0xx.org
FE_PROXY_TARGET=cast01.intra.raptor.org:80
EXPECTED_CRASHES_TABLES=16
EXPECTED_URL_SHORTENER_TABLES=4
APK_PACKAGES_BASE="nginx php83 php83-fpm php83-cli php83-session php83-pdo php83-pdo_mysql php83-pdo_sqlite php83-sqlite3 php83-mbstring php83-json php83-curl php83-openssl php83-xml php83-zip php83-phar php83-opcache mariadb-client git rsync curl bash ca-certificates sqlite wireguard-tools nfs-utils openssh-client"

22
orchestration/sim/cluster0/credentials.txt Normal file
View File

@@ -0,0 +1,22 @@
# cast cluster lab credentials (NOT production — shared intentionally for lab safety)
# Same values on cast01, cast02, cast03 unless noted.
[mariadb_root]
# Alpine default: unix socket auth as root, no password
[mariadb_app]
user=androidcast
password=cast-cluster-dev
host_write=cast01
host_read=cast01
port=3306
databases=androidcast_crashes,url_shortener
[mariadb_replication]
user=repl
password=cast-cluster-dev
allowed_from=10.7.16.%
[ssh]
user=ai
sudo=passwordless

55
orchestration/sim/cluster0/journal/README.md Normal file
View File

@@ -0,0 +1,55 @@
# Cluster setup journal (`/shared/journal/`)
Human-readable audit trail for populate / verify runs. **Read from FE** when VMs are down or after reboot failure — no SSH to cast nodes required.
## Files (per cluster)
| File | Purpose |
|------|---------|
| `{CLUSTER_NAME}.journal.tsv` | Append-only machine log (tab-separated) |
| `{CLUSTER_NAME}.summary.md` | Regenerated table with **OK** / **NOK** / **WARN** |
| `README.md` | This file |
Example for cluster0:
```
/shared/journal/cluster0.journal.tsv
/shared/journal/cluster0.summary.md
```
## TSV columns
```
timestamp_utc host actor phase action status detail
```
**status:** `START` → work begun; `OK` / `NOK` / `WARN` → outcome.
**actor:** script name (`populate`, `verify_lab`, `verify_global`, `baseline`, …).
## Who writes entries
All cluster scripts call `journal_*` helpers in `scripts/lib/common.sh`:
- `populate_lab_setup.sh` — each phase and sub-script
- `verify_lab_setup.sh` / `verify_global_setup.sh` — pass/fail
- `run_script` — every `scripts/*.sh` invocation
## Read from FE (10.7.0.10)
```sh
less /mnt/raid0/xendomains/domU_cast_cluster_0/shared/journal/cluster0.summary.md
grep NOK /mnt/raid0/xendomains/domU_cast_cluster_0/shared/journal/cluster0.journal.tsv
```
## Rebuild summary manually
```sh
sudo sh /shared/cluster/scripts/journal-rebuild-summary.sh
```
## Snapshots
If reboot breaks a node and journal shows last **OK** before power-loss, ask sysop for **cluster snapshot restore** (XEN/HVM snapshot — out of band, not stored on NFS).
Do **not** store VM disk images or MariaDB datadirs in `/shared/`.

10
orchestration/sim/cluster0/mariadb/primary.cnf Normal file
View File

@@ -0,0 +1,10 @@
[mysqld]
# cast01 — MariaDB primary (GTID async replication)
server-id=1
log_bin=mysql-bin
binlog_format=ROW
gtid_domain_id=1
log_slave_updates=1
expire_logs_days=7
bind-address=0.0.0.0
read_only=0

11
orchestration/sim/cluster0/mariadb/replica.cnf Normal file
View File

@@ -0,0 +1,11 @@
[mysqld]
# cast02/cast03 — MariaDB replica (server-id set by mariadb-replica.sh)
log_bin=mysql-bin
binlog_format=ROW
gtid_domain_id=1
log_slave_updates=1
bind-address=0.0.0.0
read_only=1
relay_log=relay-bin
replicate-do-db=androidcast_crashes
replicate-do-db=url_shortener

119
orchestration/sim/cluster0/nginx/apps-port80.conf Normal file
View File

@@ -0,0 +1,119 @@
# Full androidcast vhost for Alpine BE — listen 80 only.
# FE (apps.f0xx.org) proxies here: proxy_pass http://artc0.intra.raptor.org:80;
# Port 8089 is a different service (e.g. Janus) — not this vhost.
server {
listen 80 default_server;
listen [::]:80 default_server;
# OTA v0 static tree — devices fetch https://apps.f0xx.org/v0/ota/channel/stable.json
# Populate: builder auto_deploy, or rsync out/ota/v0/ → .../ota-artifacts/v0/
location ^~ /v0/ota/ {
alias /var/www/localhost/htdocs/apps/app/androidcast_project/ota-artifacts/v0/ota/;
add_header Cache-Control "public, max-age=60";
}
location = /app/androidcast_project {
return 301 /app/androidcast_project/;
}
location = /app/androidcast_project/index.php {
include fastcgi_params;
fastcgi_pass 127.0.0.1:9000;
fastcgi_param SCRIPT_FILENAME /var/www/localhost/htdocs/apps/app/androidcast_project/index.php;
fastcgi_param SCRIPT_NAME /app/androidcast_project/index.php;
}
location = /app/androidcast_project/ {
rewrite ^ /app/androidcast_project/index.php last;
}
location /app/androidcast_project/ {
alias /var/www/localhost/htdocs/apps/app/androidcast_project/;
index index.php index.html;
try_files $uri $uri/ =404;
}
location = /app/androidcast_project/crashes {
return 301 /app/androidcast_project/crashes/;
}
location = /app/androidcast_project/graphs {
return 301 /app/androidcast_project/graphs/;
}
location ^~ /app/androidcast_project/crashes/assets/ {
alias /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/crash_reporter/backend/public/assets/;
}
location ^~ /app/androidcast_project/graphs/assets/ {
alias /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/crash_reporter/backend/public/assets/;
}
location = /app/androidcast_project/crashes/api/upload.php {
include fastcgi_params;
fastcgi_pass 127.0.0.1:9000;
fastcgi_param SCRIPT_FILENAME /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/crash_reporter/backend/public/api/upload.php;
fastcgi_param SCRIPT_NAME /app/androidcast_project/crashes/api/upload.php;
fastcgi_param REQUEST_URI $request_uri;
client_max_body_size 4m;
}
location ^~ /app/androidcast_project/crashes/ {
include fastcgi_params;
fastcgi_pass 127.0.0.1:9000;
fastcgi_param SCRIPT_FILENAME /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/crash_reporter/backend/public/index.php;
fastcgi_param SCRIPT_NAME /app/androidcast_project/crashes/index.php;
fastcgi_param REQUEST_URI $request_uri;
client_max_body_size 4m;
}
location = /app/androidcast_project/graphs/api/graphs.php {
include fastcgi_params;
fastcgi_pass 127.0.0.1:9000;
fastcgi_param SCRIPT_FILENAME /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/crash_reporter/backend/public/api/graphs.php;
fastcgi_param SCRIPT_NAME /app/androidcast_project/graphs/api/graphs.php;
fastcgi_param REQUEST_URI $request_uri;
}
location = /app/androidcast_project/graphs/api/graph_upload.php {
include fastcgi_params;
fastcgi_pass 127.0.0.1:9000;
fastcgi_param SCRIPT_FILENAME /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/crash_reporter/backend/public/api/graph_upload.php;
fastcgi_param SCRIPT_NAME /app/androidcast_project/graphs/api/graph_upload.php;
fastcgi_param REQUEST_URI $request_uri;
client_max_body_size 4m;
}
location ^~ /app/androidcast_project/graphs/ {
include fastcgi_params;
fastcgi_pass 127.0.0.1:9000;
fastcgi_param SCRIPT_FILENAME /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/crash_reporter/backend/public/index.php;
fastcgi_param SCRIPT_NAME /app/androidcast_project/graphs/index.php;
fastcgi_param REQUEST_URI $request_uri;
client_max_body_size 4m;
}
location = /app/androidcast_project/build {
return 301 /app/androidcast_project/build/;
}
location ^~ /app/androidcast_project/build/assets/ {
alias /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/build_console/backend/public/assets/;
}
location ^~ /app/androidcast_project/build/ {
include fastcgi_params;
fastcgi_pass 127.0.0.1:9000;
fastcgi_param SCRIPT_FILENAME /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/build_console/backend/public/index.php;
fastcgi_param SCRIPT_NAME /app/androidcast_project/build/index.php;
fastcgi_param REQUEST_URI $request_uri;
client_max_body_size 512m;
fastcgi_read_timeout 1800s;
}
location ~ \.php$ {
include /etc/nginx/fastcgi.conf;
fastcgi_pass 127.0.0.1:9000;
}
}

136
orchestration/sim/cluster0/populate_lab_setup.sh Normal file
View File

@@ -0,0 +1,136 @@
#!/bin/sh
# Unattended cast cluster population — dev lab and staging template.
# Usage:
# sudo sh populate_lab_setup.sh # full setup on this node
# sudo sh populate_lab_setup.sh --coordinator # from jump host: all nodes in order
# sudo sh populate_lab_setup.sh --phase baseline|db|app|all
set -eu
ROOT="$(cd "$(dirname "$0")" && pwd)"
export CAST_CLUSTER_ROOT="$ROOT"
# shellcheck source=/dev/null
. "$ROOT/scripts/lib/common.sh"
export JOURNAL_ACTOR=populate
PHASE=all
COORDINATOR=0
while [ $# -gt 0 ]; do
case "$1" in
--coordinator) COORDINATOR=1; shift ;;
--phase) PHASE="${2:-all}"; shift 2 ;;
--help|-h)
echo "Usage: sudo sh populate_lab_setup.sh [--coordinator] [--phase baseline|db|app|all]"
exit 0
;;
*) die "unknown arg: $1" ;;
esac
done
run_phase_baseline() {
JOURNAL_PHASE=baseline
journal_start phase_baseline "$(host_short)"
log "phase baseline on $(host_short)"
journal_start apk_upgrade
apk update
apk upgrade -U -a || apk upgrade -U || true
journal_ok apk_upgrade
run_script "$ROOT/scripts/baseline.sh"
journal_ok phase_baseline
}
run_phase_db() {
JOURNAL_PHASE=db
journal_start phase_db "$(host_short)"
if is_primary_node; then
log "phase db primary on $(host_short)"
run_script "$ROOT/scripts/mariadb-primary.sh"
run_script "$ROOT/scripts/load-schemas.sh"
journal_ok phase_db "primary"
return
fi
if is_replica_node; then
log "phase db replica on $(host_short) — waiting for primary"
wait_for_primary_db
wait_for_tcp "$PRIMARY_DB_HOST" "$PRIMARY_DB_PORT" 300
run_script "$ROOT/scripts/mariadb-replica.sh"
journal_ok phase_db "replica"
return
fi
journal_nok phase_db "unknown host"
die "unknown host for db phase: $(host_short)"
}
run_phase_app() {
JOURNAL_PHASE=app
journal_start phase_app "$(host_short)"
log "phase app on $(host_short)"
if is_replica_node; then
wait_for_primary_db
fi
run_script "$ROOT/scripts/deploy-app.sh"
run_script "$ROOT/scripts/configure-nginx.sh"
rc-service php-fpm83 restart 2>/dev/null || true
rc-service nginx reload 2>/dev/null || rc-service nginx restart 2>/dev/null || true
if is_primary_node; then
rc-service mariadb restart 2>/dev/null || true
elif is_replica_node; then
rc-service mariadb restart 2>/dev/null || true
fi
journal_ok phase_app
}
run_local() {
ensure_shared_mounted
journal_start populate_local "phase=${PHASE}"
case "$PHASE" in
baseline) run_phase_baseline ;;
db) run_phase_baseline; run_phase_db ;;
app) run_phase_app ;;
all)
run_phase_baseline || exit 1
run_phase_db || exit 1
run_phase_app || exit 1
run_script "$ROOT/verify_lab_setup.sh" --local || exit 1
;;
*) journal_nok populate_local "unknown phase"; die "unknown phase: $PHASE" ;;
esac
journal_ok populate_local "phase=${PHASE}"
}
run_coordinator() {
ensure_shared_mounted
journal_start populate_coordinator "nodes=${ALL_CAST_HOSTS}"
log "coordinator: ${CLUSTER_NAME} nodes=${ALL_CAST_HOSTS}"
for H in $CAST01_HOST $CAST02_HOST $CAST03_HOST; do
ssh_node "$H" "sudo sh ${SHARED_MOUNT}/cluster/populate_lab_setup.sh --phase baseline" &
done
wait || die "baseline failed on one or more nodes"
ssh_node "$CAST01_HOST" "sudo sh ${SHARED_MOUNT}/cluster/populate_lab_setup.sh --phase db"
for H in $CAST02_HOST $CAST03_HOST; do
ssh_node "$H" "sudo sh ${SHARED_MOUNT}/cluster/populate_lab_setup.sh --phase db" &
done
wait
for H in $CAST01_HOST $CAST02_HOST $CAST03_HOST; do
ssh_node "$H" "sudo sh ${SHARED_MOUNT}/cluster/populate_lab_setup.sh --phase app" &
done
wait
sh "$ROOT/verify_lab_setup.sh" --cluster
journal_ok populate_coordinator
log "populate_coordinator_ok ${CLUSTER_NAME}"
}
if [ "$(id -u)" -ne 0 ]; then
die "run as root (sudo sh populate_lab_setup.sh)"
fi
if [ "$COORDINATOR" -eq 1 ]; then
run_coordinator
else
run_local
fi

67
orchestration/sim/cluster0/scripts/baseline.sh Normal file
View File

@@ -0,0 +1,67 @@
#!/bin/sh
# cast cluster baseline — packages, NFS, hosts, nginx/php smoke vhost.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
# shellcheck source=/dev/null
. "$ROOT/scripts/lib/common.sh"
ensure_shared_mounted
HOST="$(host_short)"
STAGE="$(mktemp -d /var/tmp/cast-cluster.XXXXXX)"
trap 'rm -rf "$STAGE"' EXIT
log "baseline start on $HOST"
grep -v "${FE_NFS_EXPORT_PATH}" /etc/fstab > "$STAGE/fstab.new" || true
echo "$FSTAB_NFS_LINE" >> "$STAGE/fstab.new"
mv "$STAGE/fstab.new" /etc/fstab
mkdir -p "$SHARED_MOUNT"
rc-update add nfsmount boot 2>/dev/null || true
mount "$SHARED_MOUNT" 2>/dev/null || mount -a 2>/dev/null || true
grep -v '# cast-cluster' /etc/hosts | grep -v 'cast0[123]\.' | grep -v 'artc0\.' > "$STAGE/hosts.new" || cp /etc/hosts "$STAGE/hosts.new"
cat >> "$STAGE/hosts.new" <<EOF
# cast-cluster ${CLUSTER_NAME}
${CAST01_IP} ${CAST01_HOST} ${CAST01_HOST}.${CAST_DOMAIN}
${CAST02_IP} ${CAST02_HOST} ${CAST02_HOST}.${CAST_DOMAIN}
${CAST03_IP} ${CAST03_HOST} ${CAST03_HOST}.${CAST_DOMAIN}
${ARTC0_IP} ${ARTC0_HOST} ${ARTC0_HOST}.${CAST_DOMAIN} alpine-be
EOF
mv "$STAGE/hosts.new" /etc/hosts
apk update
PKGS="$APK_PACKAGES_BASE"
if is_primary_node; then
PKGS="$PKGS mariadb"
fi
# shellcheck disable=SC2086
apk add --no-cache $PKGS
rc-update add nginx default 2>/dev/null || true
rc-update add php-fpm83 default 2>/dev/null || true
run_script "$ROOT/scripts/configure-php-fpm.sh"
rc-service php-fpm83 restart 2>/dev/null || rc-service php-fpm83 start 2>/dev/null || true
rc-service nginx start 2>/dev/null || true
if is_primary_node; then
grep -q '^skip-networking' /etc/my.cnf.d/mariadb-server.cnf 2>/dev/null && \
sed -i 's/^skip-networking/#skip-networking/' /etc/my.cnf.d/mariadb-server.cnf || true
grep -q '^bind-address' /etc/my.cnf.d/mariadb-server.cnf 2>/dev/null || \
printf '\n[mysqld]\nbind-address = 0.0.0.0\n' >> /etc/my.cnf.d/mariadb-server.cnf
rc-update add mariadb default 2>/dev/null || true
rc-service mariadb restart 2>/dev/null || rc-service mariadb start 2>/dev/null || true
fi
mkdir -p /var/www/localhost/htdocs
echo "<html><body><h1>${HOST}</h1><p>${CLUSTER_NAME} node</p></body></html>" > /var/www/localhost/htdocs/index.html
chown -R nginx:nginx /var/www/localhost/htdocs 2>/dev/null || true
mkdir -p "$ROOT"
IP="$(ip -4 -o addr show eth0 2>/dev/null | awk '{print $4}' | cut -d/ -f1)"
echo "$HOST $(date -Iseconds) ip=$IP cluster=$CLUSTER_NAME" >> "$ROOT/seen.log"
printf '{"host":"%s","ip":"%s","cluster":"%s","configured_at":"%s"}\n' \
"$HOST" "$IP" "$CLUSTER_NAME" "$(date -Iseconds)" > "$ROOT/${HOST}.json"
mount | grep -q " on ${SHARED_MOUNT} " || die "NFS not mounted"
curl -sS -o /dev/null -w "nginx_http=%{http_code}\n" http://127.0.0.1/
log "baseline_ok $HOST"

12
orchestration/sim/cluster0/scripts/configure-nginx.sh Normal file
View File

@@ -0,0 +1,12 @@
#!/bin/sh
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
# shellcheck source=/dev/null
. "$ROOT/scripts/lib/common.sh"
ensure_shared_mounted
install -D -m 644 "$ROOT/nginx/apps-port80.conf" /etc/nginx/http.d/androidcast.conf
rm -f /etc/nginx/http.d/default.conf /etc/nginx/http.d/cluster.conf 2>/dev/null || true
nginx -t
rc-service nginx reload
log "nginx androidcast vhost installed"

13
orchestration/sim/cluster0/scripts/configure-php-fpm.sh Normal file
View File

@@ -0,0 +1,13 @@
#!/bin/sh
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
# shellcheck source=/dev/null
. "$ROOT/scripts/lib/common.sh"
ensure_shared_mounted
CONF=/etc/php83/php-fpm.d/www.conf
grep -q '^listen = 127.0.0.1:9000' "$CONF" 2>/dev/null || \
sed -i 's|^listen = .*|listen = 127.0.0.1:9000|' "$CONF"
grep -q '^listen.allowed_clients' "$CONF" 2>/dev/null || \
printf '\nlisten.allowed_clients = 127.0.0.1\n' >> "$CONF"
log "php-fpm83 listen 127.0.0.1:9000"

109
orchestration/sim/cluster0/scripts/deploy-app.sh Normal file
View File

@@ -0,0 +1,109 @@
#!/bin/sh
# Deploy androidcast project tree + lab config.php (local disk, not NFS).
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
# shellcheck source=/dev/null
. "$ROOT/scripts/lib/common.sh"
ensure_shared_mounted
APP_PASS="$(read_cred mariadb_app password)"
DB_HOST="$PRIMARY_DB_HOST"
if is_primary_node; then
DB_HOST=127.0.0.1
fi
mkdir -p "$(dirname "$APP_ROOT")"
GIT_ROOT="${APP_ROOT}/android_cast"
BACKEND="${GIT_ROOT}/examples/crash_reporter/backend"
CONFIG="${BACKEND}/config/config.php"
case "$APP_SOURCE" in
git)
if [ -d "${GIT_ROOT}/.git" ]; then
log "git pull ${GIT_BRANCH} in ${GIT_ROOT}"
git config --global --add safe.directory "$GIT_ROOT" 2>/dev/null || true
git -C "$GIT_ROOT" fetch origin
git -C "$GIT_ROOT" checkout "$GIT_BRANCH"
git -C "$GIT_ROOT" pull --ff-only origin "$GIT_BRANCH"
else
log "git clone ${GIT_ORIGIN} → ${GIT_ROOT}"
rm -rf "$GIT_ROOT"
git clone --branch "$GIT_BRANCH" --depth 1 "$GIT_ORIGIN" "$GIT_ROOT"
git config --global --add safe.directory "$GIT_ROOT" 2>/dev/null || true
fi
;;
bundle)
BUNDLE="${ROOT}/release/current"
[ -d "$BUNDLE" ] || die "missing bundle dir $BUNDLE (APP_SOURCE=bundle)"
log "rsync bundle $BUNDLE → ${APP_ROOT}"
mkdir -p "$APP_ROOT"
rsync -a --delete "$BUNDLE/" "$APP_ROOT/"
;;
*)
die "unknown APP_SOURCE=$APP_SOURCE"
;;
esac
mkdir -p "${BACKEND}/config" "${BACKEND}/data" "${BACKEND}/storage"
chown -R nginx:nginx "$APP_ROOT" 2>/dev/null || true
if [ ! -f "$CONFIG" ] || [ "${FORCE_CONFIG:-1}" = "1" ]; then
log "writing lab config.php"
cat > "$CONFIG" <<PHP
<?php
return [
'app_name' => 'Android Cast Issues',
'base_path' => '${APP_BASE_PATH}',
'db' => [
'driver' => 'mariadb',
'sqlite_path' => __DIR__ . '/../data/crashes.sqlite',
'mysql' => [
'host' => '${DB_HOST}',
'port' => ${PRIMARY_DB_PORT},
'socket' => '',
'database' => 'androidcast_crashes',
'username' => 'androidcast',
'password' => '${APP_PASS}',
'charset' => 'utf8mb4',
],
],
'session_name' => 'ac_crash_sess',
'session_cookie_path' => '${APP_HUB_PATH}',
'debug' => true,
'rbac' => [
'default_company_slug' => 'default',
'default_company_id' => 1,
],
'analytics' => ['enabled' => false, 'measurement_id' => '', 'debug' => false],
'remote_access' => [
'wg_endpoint' => '',
'wg_server_public_key' => '',
'provision_peers' => false,
'require_wg_tools' => false,
],
'auth' => [
'encryption_key' => 'lab-cluster-dev-32-char-min-secret!!',
'max_attempts' => 8,
'lockout_window_minutes' => 15,
],
'url_shortener' => [
'enabled' => true,
'public_base' => '${SHORT_LINKS_PUBLIC_BASE}',
'db' => [
'mysql' => [
'host' => '${DB_HOST}',
'port' => ${PRIMARY_DB_PORT},
'socket' => '',
'database' => 'url_shortener',
'username' => 'androidcast',
'password' => '${APP_PASS}',
'charset' => 'utf8mb4',
],
],
],
];
PHP
chown nginx:nginx "$CONFIG" 2>/dev/null || true
fi
log "deploy-app_ok $(host_short)"

57
orchestration/sim/cluster0/scripts/journal-backfill-cluster0.sh Normal file
View File

@@ -0,0 +1,57 @@
#!/bin/sh
# One-time backfill of cluster0 journal from known-good state (2026-06-13 session).
# Safe to re-run: appends only if journal is empty.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
# shellcheck source=/dev/null
. "$ROOT/scripts/lib/common.sh"
ensure_shared_mounted
journal_init
if [ -f "$JOURNAL_TSV" ] && [ "$(wc -l < "$JOURNAL_TSV")" -gt 1 ]; then
log "journal already has entries — skip backfill (see $JOURNAL_TSV)"
sh "$ROOT/scripts/journal-rebuild-summary.sh"
exit 0
fi
export JOURNAL_ACTOR=backfill
TS="2026-06-13T12:00:00+00:00"
append() {
_host="$1"
_phase="$2"
_action="$3"
_status="$4"
_detail="$5"
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
"$TS" "$_host" "$JOURNAL_ACTOR" "$_phase" "$_action" "$_status" "$_detail" >> "$JOURNAL_TSV"
}
# Infrastructure (all nodes)
for H in cast01 cast02 cast03; do
append "$H" baseline nfs_mount OK "/shared from FE export"
append "$H" baseline packages OK "nginx php83-fpm mariadb-client nfs-utils"
append "$H" baseline hosts OK "cast-cluster /etc/hosts block"
done
# MariaDB
append cast01 db mariadb-primary OK "GTID primary server-id=1"
append cast01 db load-schemas OK "16 crashes + 4 url_shortener tables"
append cast02 db mariadb-replica OK "GTID replica IO+SQL running"
append cast03 db mariadb-replica OK "GTID replica IO+SQL running"
# App + nginx
for H in cast01 cast02 cast03; do
append "$H" app deploy-app OK "git next → android_cast/"
append "$H" app configure-nginx OK "apps-port80.conf php83:9000"
done
# Verification
append cast01 verify verify_lab OK "mode=local"
append cast02 verify verify_lab OK "mode=local"
append cast03 verify verify_lab OK "mode=local"
append cast01 verify verify_lab OK "mode=cluster HTTP peers"
append cast01 verify verify_global START "pending DNS/FE cutover"
sh "$ROOT/scripts/journal-rebuild-summary.sh"
log "backfill_ok → $JOURNAL_TSV"

35
orchestration/sim/cluster0/scripts/journal-rebuild-summary.sh Normal file
View File

@@ -0,0 +1,35 @@
#!/bin/sh
# Regenerate human-readable OK/NOK summary from journal TSV (safe on FE via NFS read).
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
# shellcheck source=/dev/null
. "$ROOT/scripts/lib/common.sh"
load_cluster_env
journal_init
{
echo "# ${CLUSTER_NAME} — setup journal summary"
echo
echo "Read raw log on FE: \`${JOURNAL_TSV}\`"
echo
echo "Last rebuilt: $(date -u -Iseconds 2>/dev/null || date -u)"
echo
echo "| UTC time | Host | Phase | Action | Status | Detail |"
echo "|----------|------|-------|--------|--------|--------|"
tail -n +2 "$JOURNAL_TSV" 2>/dev/null | while IFS=' ' read -r ts host actor phase action status detail; do
printf '| %s | %s | %s | %s | **%s** | %s |\n' \
"$ts" "$host" "$phase" "$action" "$status" "$detail"
done
echo
echo "## Latest status per action/host"
echo
tail -n +2 "$JOURNAL_TSV" 2>/dev/null | awk -F' ' '
{ key=$2 SUBSEP $5 SUBSEP $4; line=$0; status=$6; if (status != "START") last[key]=line }
END { for (k in last) print last[k] }' | sort -t' ' -k1,1 | while IFS=' ' read -r ts host actor phase action status detail; do
printf -- '- **%s** `%s` / `%s`' "$status" "$host" "$action"
[ -n "$detail" ] && printf ' — %s' "$detail"
printf '\n'
done
} > "$JOURNAL_SUMMARY"
echo "summary → $JOURNAL_SUMMARY"

192
orchestration/sim/cluster0/scripts/lib/common.sh Normal file
View File

@@ -0,0 +1,192 @@
# shellcheck shell=sh
# Shared helpers for cast cluster scripts.
log() {
printf '[%s] %s\n' "$(date -Iseconds 2>/dev/null || date)" "$*"
}
die() {
log "ERROR: $*"
if [ -n "${JOURNAL_LAST_ACTION:-}" ]; then
journal_record NOK "$JOURNAL_LAST_ACTION" "$*"
fi
exit 1
}
cluster_root() {
if [ -n "${CAST_CLUSTER_ROOT:-}" ]; then
printf '%s\n' "$CAST_CLUSTER_ROOT"
return
fi
if [ -f /shared/cluster/cluster.env ]; then
printf '/shared/cluster\n'
return
fi
if [ -f "$(dirname "$0")/../cluster.env" ]; then
cd "$(dirname "$0")/.." && pwd
return
fi
die "cannot locate cluster root (is /shared mounted?)"
}
load_cluster_env() {
ROOT="$(cluster_root)"
ENV_FILE="${1:-$ROOT/cluster.env}"
[ -f "$ENV_FILE" ] || die "missing $ENV_FILE"
# shellcheck disable=SC1090
. "$ENV_FILE"
CAST_CLUSTER_ROOT="$ROOT"
FE_NFS="${FE_NFS_SERVER}:${FE_NFS_EXPORT_PATH}"
FSTAB_NFS_LINE="${FE_NFS} ${SHARED_MOUNT} nfs rw,_netdev,nofail,noatime,nodiratime 0 0"
ALL_CAST_HOSTS="${CAST01_HOST} ${CAST02_HOST} ${CAST03_HOST}"
JOURNAL_DIR="${SHARED_MOUNT}/journal"
JOURNAL_TSV="${JOURNAL_DIR}/${CLUSTER_NAME}.journal.tsv"
JOURNAL_SUMMARY="${JOURNAL_DIR}/${CLUSTER_NAME}.summary.md"
}
journal_init() {
load_cluster_env 2>/dev/null || return 0
mkdir -p "$JOURNAL_DIR"
if [ ! -f "$JOURNAL_TSV" ]; then
printf 'timestamp_utc\thost\tactor\tphase\taction\tstatus\tdetail\n' > "$JOURNAL_TSV"
fi
}
journal_record() {
_status="$1"
_action="$2"
_detail="${3:-}"
_phase="${JOURNAL_PHASE:-}"
journal_init
_ts="$(date -u -Iseconds 2>/dev/null || date -u +%Y-%m-%dT%H:%M:%SZ)"
_host="$(host_short 2>/dev/null || echo unknown)"
_actor="${JOURNAL_ACTOR:-script}"
_line="${_ts} ${_host} ${_actor} ${_phase} ${_action} ${_status} ${_detail}"
printf '%s\n' "$_line" >> "$JOURNAL_TSV"
log "journal ${_status} ${_action}${_detail:+ — ${_detail}}"
if [ -x "${CAST_CLUSTER_ROOT}/scripts/journal-rebuild-summary.sh" ] 2>/dev/null; then
sh "${CAST_CLUSTER_ROOT}/scripts/journal-rebuild-summary.sh" 2>/dev/null || true
elif [ -f "${CAST_CLUSTER_ROOT}/scripts/journal-rebuild-summary.sh" ]; then
sh "${CAST_CLUSTER_ROOT}/scripts/journal-rebuild-summary.sh" 2>/dev/null || true
fi
}
journal_start() {
JOURNAL_LAST_ACTION="$1"
journal_record START "$1" "${2:-}"
}
journal_ok() {
journal_record OK "$1" "${2:-}"
JOURNAL_LAST_ACTION=""
}
journal_nok() {
journal_record NOK "$1" "${2:-}"
JOURNAL_LAST_ACTION=""
}
journal_warn() {
journal_record WARN "$1" "${2:-}"
}
ensure_shared_mounted() {
load_cluster_env
if mount | grep -q " on ${SHARED_MOUNT} "; then
journal_init
return 0
fi
if [ -f /etc/cast-cluster.env ]; then
# shellcheck disable=SC1091
. /etc/cast-cluster.env
FE_NFS="${FE_NFS_SERVER}:${FE_NFS_EXPORT_PATH}"
FSTAB_NFS_LINE="${FE_NFS} ${SHARED_MOUNT} nfs rw,_netdev,nofail,noatime,nodiratime 0 0"
fi
log "mounting ${SHARED_MOUNT} from ${FE_NFS}"
journal_start nfs_mount "from ${FE_NFS}"
mkdir -p "$SHARED_MOUNT"
grep -v "${FE_NFS_EXPORT_PATH}" /etc/fstab > /var/tmp/cast-fstab.new 2>/dev/null || cp /etc/fstab /var/tmp/cast-fstab.new
echo "$FSTAB_NFS_LINE" >> /var/tmp/cast-fstab.new
mv /var/tmp/cast-fstab.new /etc/fstab
rc-update add nfsmount boot 2>/dev/null || true
mount "$SHARED_MOUNT" 2>/dev/null || mount -a
if mount | grep -q " on ${SHARED_MOUNT} "; then
load_cluster_env
journal_ok nfs_mount "${SHARED_MOUNT}"
else
journal_nok nfs_mount "mount failed"
die "NFS mount failed: ${SHARED_MOUNT}"
fi
}
run_script() {
_script="$1"
_action="$(basename "$_script" .sh)"
journal_start "$_action" "$_script"
if sh "$_script"; then
journal_ok "$_action" "$_script"
return 0
fi
journal_nok "$_action" "$_script"
return 1
}
host_short() {
hostname -s 2>/dev/null || hostname
}
is_primary_node() {
[ "$(host_short)" = "$CAST01_HOST" ]
}
is_replica_node() {
H="$(host_short)"
[ "$H" = "$CAST02_HOST" ] || [ "$H" = "$CAST03_HOST" ]
}
wait_for_tcp() {
_host="$1"
_port="$2"
_timeout="${3:-300}"
_i=0
while [ "$_i" -lt "$_timeout" ]; do
if nc -z "$_host" "$_port" 2>/dev/null; then
return 0
fi
_i=$((_i + 1))
sleep 1
done
die "timeout waiting for ${_host}:${_port}"
}
wait_for_primary_db() {
load_cluster_env
CREDS="${CAST_CLUSTER_ROOT}/credentials.txt"
APP_PASS="$(awk '/^\[mariadb_app\]/{f=1;next} /^\[/{f=0} f&&/^password=/{print substr($0,10); exit}' "$CREDS")"
_i=0
while [ "$_i" -lt 300 ]; do
if mariadb -u androidcast -p"$APP_PASS" -h"$PRIMARY_DB_HOST" -N -e 'SELECT 1' >/dev/null 2>&1; then
return 0
fi
_i=$((_i + 1))
sleep 2
done
die "primary DB not reachable at ${PRIMARY_DB_HOST}"
}
read_cred() {
_section="$1"
_key="$2"
_file="${CAST_CLUSTER_ROOT}/credentials.txt"
awk -v s="[$_section]" -v k="$_key=" '
$0 == s { f=1; next }
/^\[/ { f=0 }
f && index($0, k) == 1 { print substr($0, length(k)+1); exit }
' "$_file"
}
ssh_node() {
_host="$1"
shift
ssh -o BatchMode=yes -o StrictHostKeyChecking=no "${SSH_USER}@${_host}" "$@"
}

39
orchestration/sim/cluster0/scripts/load-schemas.sh Normal file
View File

@@ -0,0 +1,39 @@
#!/bin/sh
# Load project schemas on cast01 primary only.
set -eu
SHARED=/shared/cluster/sql
[ "$(hostname -s)" = cast01 ] || { echo "run on cast01 only"; exit 1; }
apply() {
f="$1"
[ -f "$f" ] || { echo "missing $f"; exit 1; }
echo "==> $f"
mariadb -u root < "$f"
}
apply_db() {
db="$1"
f="$2"
echo "==> $db < $f"
mariadb -u root "$db" < "$f"
}
apply "$SHARED/crashes/schema.mariadb.sql"
# Migrations after base schema (003 is for legacy; base schema already includes tickets)
for m in \
004_ticket_workflow.sql \
005_ticket_attachments.sql \
006_graph_sessions.sql \
007_remote_access.sql \
008_graph_sessions_indexes.sql \
008_auth_email_2fa.sql \
009_rssh_sessions.sql
do
apply_db androidcast_crashes "$SHARED/crashes/migrations/$m"
done
apply "$SHARED/url_shortener/schema.mariadb.sql"
mariadb -u root -e "SHOW DATABASES; SELECT COUNT(*) AS crash_tables FROM information_schema.tables WHERE table_schema='androidcast_crashes'; SELECT COUNT(*) AS url_tables FROM information_schema.tables WHERE table_schema='url_shortener';"
echo "schemas_loaded_ok"

33
orchestration/sim/cluster0/scripts/mariadb-primary.sh Normal file
View File

@@ -0,0 +1,33 @@
#!/bin/sh
# Configure cast01 as MariaDB GTID primary. Idempotent-ish.
set -eu
SHARED=/shared/cluster
CREDS="$SHARED/credentials.txt"
REPL_USER="$(awk '/^\[mariadb_replication\]/{f=1;next} /^\[/{f=0} f&&/^user=/{print substr($0,6); exit}' "$CREDS")"
REPL_PASS="$(awk '/^\[mariadb_replication\]/{f=1;next} /^\[/{f=0} f&&/^password=/{print substr($0,10); exit}' "$CREDS")"
APP_USER="$(awk '/^\[mariadb_app\]/{f=1;next} /^\[/{f=0} f&&/^user=/{print substr($0,6); exit}' "$CREDS")"
APP_PASS="$(awk '/^\[mariadb_app\]/{f=1;next} /^\[/{f=0} f&&/^password=/{print substr($0,10); exit}' "$CREDS")"
[ "$(hostname -s)" = cast01 ] || { echo "run on cast01 only"; exit 1; }
grep -q '^skip-networking' /etc/my.cnf.d/mariadb-server.cnf && \
sed -i 's/^skip-networking/#skip-networking/' /etc/my.cnf.d/mariadb-server.cnf
install -D -m 644 "$SHARED/mariadb/primary.cnf" /etc/my.cnf.d/lab-cluster-primary.cnf
rc-update add mariadb default 2>/dev/null || true
rc-service mariadb restart
mariadb -u root <<SQL
CREATE USER IF NOT EXISTS '${REPL_USER}'@'10.7.16.%' IDENTIFIED BY '${REPL_PASS}';
GRANT REPLICATION SLAVE, REPLICATION CLIENT, RELOAD, LOCK TABLES, SELECT, SHOW VIEW, EVENT, TRIGGER ON *.* TO '${REPL_USER}'@'10.7.16.%';
CREATE USER IF NOT EXISTS '${APP_USER}'@'10.7.16.%' IDENTIFIED BY '${APP_PASS}';
CREATE USER IF NOT EXISTS '${APP_USER}'@'localhost' IDENTIFIED BY '${APP_PASS}';
GRANT ALL PRIVILEGES ON androidcast_crashes.* TO '${APP_USER}'@'10.7.16.%';
GRANT ALL PRIVILEGES ON androidcast_crashes.* TO '${APP_USER}'@'localhost';
GRANT ALL PRIVILEGES ON url_shortener.* TO '${APP_USER}'@'10.7.16.%';
GRANT ALL PRIVILEGES ON url_shortener.* TO '${APP_USER}'@'localhost';
FLUSH PRIVILEGES;
SQL
echo "primary_ok server_id=$(mariadb -u root -N -e 'SELECT @@server_id')"
echo "gtid=$(mariadb -u root -N -e 'SELECT @@gtid_current_pos')"

73
orchestration/sim/cluster0/scripts/mariadb-replica.sh Normal file
View File

@@ -0,0 +1,73 @@
#!/bin/sh
# Bootstrap cast02 or cast03 as GTID replica of cast01.
set -eu
set -o pipefail
SHARED=/shared/cluster
CREDS="$SHARED/credentials.txt"
[ -f "$SHARED/cluster.env" ] && . "$SHARED/cluster.env"
HOST="$(hostname -s)"
case "$HOST" in
cast02) SERVER_ID=2 ;;
cast03) SERVER_ID=3 ;;
*) echo "run on cast02 or cast03"; exit 1 ;;
esac
REPL_PASS="$(awk '/^\[mariadb_replication\]/{f=1;next} /^\[/{f=0} f&&/^password=/{print substr($0,10); exit}' "$CREDS")"
REPL_USER="$(awk '/^\[mariadb_replication\]/{f=1;next} /^\[/{f=0} f&&/^user=/{print substr($0,6); exit}' "$CREDS")"
APP_PASS="$(awk '/^\[mariadb_app\]/{f=1;next} /^\[/{f=0} f&&/^password=/{print substr($0,10); exit}' "$CREDS")"
APP_USER="$(awk '/^\[mariadb_app\]/{f=1;next} /^\[/{f=0} f&&/^user=/{print substr($0,6); exit}' "$CREDS")"
grep -q '^skip-networking' /etc/my.cnf.d/mariadb-server.cnf && \
sed -i 's/^skip-networking/#skip-networking/' /etc/my.cnf.d/mariadb-server.cnf
apk add --no-cache mariadb
install -D -m 644 "$SHARED/mariadb/replica.cnf" /etc/my.cnf.d/lab-cluster-replica.cnf
grep -q '^server-id=' /etc/my.cnf.d/lab-cluster-replica.cnf && \
sed -i "s/^server-id=.*/server-id=${SERVER_ID}/" /etc/my.cnf.d/lab-cluster-replica.cnf || \
echo "server-id=${SERVER_ID}" >> /etc/my.cnf.d/lab-cluster-replica.cnf
rc-service mariadb stop 2>/dev/null || true
rm -rf /var/lib/mysql/*
/etc/init.d/mariadb setup
rc-update add mariadb default
rc-service mariadb restart
sleep 2
echo "=== app DB sync from cast01 (skip mysql.* — avoids help_topic dump failures) ==="
mariadb-dump --skip-ssl -h cast01 -u "${REPL_USER}" -p"${REPL_PASS}" \
--databases androidcast_crashes url_shortener \
--single-transaction --gtid --routines --events \
| mariadb -u root
GTID_POS="$(mariadb --skip-ssl -h cast01 -u "${REPL_USER}" -p"${REPL_PASS}" -N -e 'SELECT @@gtid_current_pos')"
[ -n "$GTID_POS" ] || { echo "FAIL: could not read gtid from cast01"; exit 1; }
mariadb -u root -e "SET GLOBAL gtid_slave_pos='${GTID_POS}'"
echo "gtid_slave_pos set to: ${GTID_POS}"
mariadb -u root <<SQL
CREATE USER IF NOT EXISTS '${APP_USER}'@'10.7.16.%' IDENTIFIED BY '${APP_PASS}';
CREATE USER IF NOT EXISTS '${APP_USER}'@'localhost' IDENTIFIED BY '${APP_PASS}';
GRANT ALL PRIVILEGES ON androidcast_crashes.* TO '${APP_USER}'@'10.7.16.%';
GRANT ALL PRIVILEGES ON androidcast_crashes.* TO '${APP_USER}'@'localhost';
GRANT ALL PRIVILEGES ON url_shortener.* TO '${APP_USER}'@'10.7.16.%';
GRANT ALL PRIVILEGES ON url_shortener.* TO '${APP_USER}'@'localhost';
FLUSH PRIVILEGES;
CHANGE MASTER TO
MASTER_HOST='cast01',
MASTER_USER='${REPL_USER}',
MASTER_PASSWORD='${REPL_PASS}',
MASTER_USE_GTID=slave_pos;
START REPLICA;
SQL
sleep 3
mariadb -u root -e "SHOW REPLICA STATUS\G" | grep -E 'Slave_IO_Running|Slave_SQL_Running|Last_Error|Last_SQL_Error|Seconds_Behind_Master'
mariadb -u root -e "SELECT table_schema, COUNT(*) AS tables FROM information_schema.tables WHERE table_schema IN ('androidcast_crashes','url_shortener') GROUP BY table_schema;"
URL_N="$(mariadb -u root -N -e 'SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='"'"'url_shortener'"'"'')"
CRASH_N="$(mariadb -u root -N -e 'SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='"'"'androidcast_crashes'"'"'')"
EXP_CRASH="${EXPECTED_CRASHES_TABLES:-16}"
EXP_URL="${EXPECTED_URL_SHORTENER_TABLES:-4}"
[ "$CRASH_N" = "$EXP_CRASH" ] && [ "$URL_N" = "$EXP_URL" ] || { echo "FAIL: expected ${EXP_CRASH}+${EXP_URL} tables, got crashes=$CRASH_N url=$URL_N"; exit 1; }
mariadb -u root -e "SHOW REPLICA STATUS\G" | grep -E 'Slave_SQL_Running: Yes' >/dev/null || { echo "FAIL: SQL thread not running"; mariadb -u root -e "SHOW REPLICA STATUS\G" | grep -E 'Last_SQL_Error'; exit 1; }
echo "replica_ok $HOST server_id=$SERVER_ID"

34
orchestration/sim/cluster0/scripts/verify-cluster.sh Normal file
View File

@@ -0,0 +1,34 @@
#!/bin/sh
# Quick health check for cast cluster.
set -eu
HOST="$(hostname -s)"
FAIL=0
echo "=== $HOST ==="
mount | grep ' /shared ' || { echo "WARN: /shared not mounted"; FAIL=1; }
curl -sS -o /dev/null -w "nginx=%{http_code}\n" http://127.0.0.1/ || true
php83 -v | head -1
case "$HOST" in
cast01)
mariadb -u root -e "SELECT @@server_id AS server_id, @@read_only AS read_only, @@gtid_current_pos AS gtid;"
mariadb -u androidcast -pcast-cluster-dev -h127.0.0.1 -e \
"SELECT table_schema, COUNT(*) AS tables FROM information_schema.tables WHERE table_schema IN ('androidcast_crashes','url_shortener') GROUP BY table_schema;"
;;
cast02|cast03)
mariadb -u root -e "SELECT @@server_id AS server_id, @@read_only AS read_only;" 2>/dev/null || { echo "mariadb not running"; FAIL=1; }
RS="$(mariadb -u root -e 'SHOW REPLICA STATUS\G' 2>/dev/null || true)"
echo "$RS" | grep -E 'Master_Host|Slave_IO_Running|Slave_SQL_Running|Seconds_Behind_Master|Last_SQL_Error' || true
echo "$RS" | grep -E 'Slave_IO_Running: Yes' >/dev/null || { echo "FAIL: IO thread not running"; FAIL=1; }
echo "$RS" | grep -E 'Slave_SQL_Running: Yes' >/dev/null || { echo "FAIL: SQL thread not running"; FAIL=1; }
mariadb -u root -e \
"SELECT table_schema, COUNT(*) AS tables FROM information_schema.tables WHERE table_schema IN ('androidcast_crashes','url_shortener') GROUP BY table_schema;"
mariadb -u androidcast -pcast-cluster-dev -h127.0.0.1 -e "SELECT 1 AS local_read_ok;" 2>/dev/null || echo "WARN: local app user read failed"
;;
esac
if [ "$FAIL" -eq 0 ]; then
echo OK
else
echo FAIL
exit 1
fi

64
orchestration/sim/cluster0/verify_global_setup.sh Normal file
View File

@@ -0,0 +1,64 @@
#!/bin/sh
# Post-DNS / post-FE verification — run after PUBLIC_ORIGIN points at this cluster.
set -eu
ROOT="$(cd "$(dirname "$0")" && pwd)"
export CAST_CLUSTER_ROOT="$ROOT"
# shellcheck source=/dev/null
. "$ROOT/scripts/lib/common.sh"
export JOURNAL_ACTOR=verify_global
export JOURNAL_PHASE=verify_global
ensure_shared_mounted
FAIL=0
ORIGIN="${PUBLIC_ORIGIN%/}"
check_url() {
_path="$1"
_expect="${2:-200}"
_url="${ORIGIN}${_path}"
_action="curl ${_path}"
journal_start "$_action" "$_url"
_code="$(curl -sS -o /dev/null -w '%{http_code}' -L --max-time 30 "$_url" 2>/dev/null || echo 000)"
if [ "$_code" = "$_expect" ]; then
journal_ok "$_action" "http=${_code}"
log "OK $_code $_url"
else
journal_nok "$_action" "got=${_code} want=${_expect}"
log "FAIL got $_code want $_expect $_url"
FAIL=1
fi
}
journal_start verify_global "origin=${ORIGIN}"
log "verify_global_setup cluster=${CLUSTER_NAME} origin=${ORIGIN}"
[ -n "$ORIGIN" ] || die "set PUBLIC_ORIGIN in cluster.env"
check_url "${APP_HUB_PATH}/" "200"
check_url "${APP_BASE_PATH}/" "200"
check_url "${APP_BASE_PATH}/assets/js/app.js" "200"
check_url "${APP_BASE_PATH}/api/diag.php" "200"
# Short links UI (may redirect to login)
_code="$(curl -sS -o /dev/null -w '%{http_code}' -L --max-time 30 "${ORIGIN}${APP_BASE_PATH}/?view=short_links" 2>/dev/null || echo 000)"
case "$_code" in
200|302) log "OK $_code short_links" ;;
*) log "FAIL short_links http=$_code"; FAIL=1 ;;
esac
if [ -n "${FE_PROXY_TARGET:-}" ]; then
_fe_code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 15 "http://${FE_PROXY_TARGET}${APP_BASE_PATH}/" 2>/dev/null || echo 000)"
case "$_fe_code" in
200|302) log "OK upstream $_fe_code http://${FE_PROXY_TARGET}${APP_BASE_PATH}/" ;;
*) log "WARN upstream $_fe_code (check FE nginx after DNS cutover)" ;;
esac
fi
if [ "$FAIL" -eq 0 ]; then
journal_ok verify_global
log "verify_global_setup_ok ${CLUSTER_NAME}"
exit 0
fi
journal_nok verify_global
log "verify_global_setup_FAIL ${CLUSTER_NAME}"
exit 1

105
orchestration/sim/cluster0/verify_lab_setup.sh Normal file
View File

@@ -0,0 +1,105 @@
#!/bin/sh
# Internal cluster verification (before or after DNS). No external network required for --local.
set -eu
ROOT="$(cd "$(dirname "$0")" && pwd)"
export CAST_CLUSTER_ROOT="$ROOT"
# shellcheck source=/dev/null
. "$ROOT/scripts/lib/common.sh"
export JOURNAL_ACTOR=verify_lab
export JOURNAL_PHASE=verify
MODE=local
while [ $# -gt 0 ]; do
case "$1" in
--local|--local-only) MODE=local; shift ;;
--cluster) MODE=cluster; shift ;;
*) die "unknown arg: $1" ;;
esac
done
FAIL=0
check_local_node() {
H="$(host_short)"
log "verify local $H"
run_script "$ROOT/scripts/verify-cluster.sh" || FAIL=1
nginx -t >/dev/null 2>&1 || { log "FAIL nginx -t"; FAIL=1; }
rc-status nginx php-fpm83 2>/dev/null | grep -E 'nginx|php-fpm83' || true
if is_primary_node || is_replica_node; then
rc-service mariadb status 2>/dev/null | grep -q started || { log "FAIL mariadb not running"; FAIL=1; }
fi
CODE="$(curl -sS -o /dev/null -w '%{http_code}' "http://127.0.0.1${APP_BASE_PATH}/" 2>/dev/null || echo 000)"
case "$CODE" in
200|302) log "app_http=${CODE}" ;;
*) log "FAIL app_http=${CODE} for ${APP_BASE_PATH}/"; FAIL=1 ;;
esac
ASSET="${APP_BASE_PATH}/assets/js/app.js"
ACODE="$(curl -sS -o /dev/null -w '%{http_code}' "http://127.0.0.1${ASSET}" 2>/dev/null || echo 000)"
case "$ACODE" in
200) log "asset_http=${ACODE}" ;;
*) log "WARN asset_http=${ACODE} (${ASSET})" ;;
esac
DIAG="$(curl -sS "http://127.0.0.1${APP_BASE_PATH}/api/diag.php" 2>/dev/null || true)"
echo "$DIAG" | grep -q '"ok":true' || { log "FAIL diag.php"; FAIL=1; }
CRASH_N="$(mariadb -u androidcast -p"$(read_cred mariadb_app password)" -h127.0.0.1 -N -e \
"SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='androidcast_crashes'" 2>/dev/null || echo 0)"
[ "$CRASH_N" = "$EXPECTED_CRASHES_TABLES" ] || { log "FAIL crashes tables=$CRASH_N want $EXPECTED_CRASHES_TABLES"; FAIL=1; }
}
check_remote_node_http() {
_host="$1"
_ip=""
case "$_host" in
"$CAST01_HOST") _ip="$CAST01_IP" ;;
"$CAST02_HOST") _ip="$CAST02_IP" ;;
"$CAST03_HOST") _ip="$CAST03_IP" ;;
esac
[ -n "$_ip" ] || { log "FAIL unknown host $_host"; FAIL=1; return; }
log "verify http $_host ($_ip)"
_code="$(curl -sS -o /dev/null -w '%{http_code}' "http://${_ip}${APP_BASE_PATH}/" 2>/dev/null || echo 000)"
case "$_code" in
200|302) log "OK $_host app_http=$_code" ;;
*) log "FAIL $_host app_http=$_code"; FAIL=1 ;;
esac
_diag="$(curl -sS "http://${_ip}${APP_BASE_PATH}/api/diag.php" 2>/dev/null || true)"
echo "$_diag" | grep -q '"ok":true' || { log "FAIL $_host diag.php"; FAIL=1; }
}
check_remote_node_ssh() {
_host="$1"
log "verify ssh $_host"
ssh_node "$_host" "sudo sh ${SHARED_MOUNT}/cluster/verify_lab_setup.sh --local" || FAIL=1
}
ensure_shared_mounted
journal_start verify_lab "mode=${MODE}"
if [ "$MODE" = cluster ]; then
for H in $CAST01_HOST $CAST02_HOST $CAST03_HOST; do
if [ "$(host_short)" = "$H" ]; then
check_local_node
else
if ssh -o BatchMode=yes -o ConnectTimeout=3 "${SSH_USER}@${H}" true 2>/dev/null; then
check_remote_node_ssh "$H"
else
check_remote_node_http "$H"
fi
fi
done
else
check_local_node
fi
if [ "$FAIL" -eq 0 ]; then
journal_ok verify_lab "mode=${MODE}"
log "verify_lab_setup_ok ${CLUSTER_NAME} mode=$MODE"
exit 0
fi
journal_nok verify_lab "mode=${MODE}"
log "verify_lab_setup_FAIL ${CLUSTER_NAME} mode=$MODE"
exit 1