sync on be, vpn

2026-07-29 05:37:52 +03:00 · 2026-06-13 15:31:18 +02:00
parent f9b73b48da
commit 86055bf509
42 changed files with 2472 additions and 917 deletions
--- a/orchestration/sim/cluster0/ARCHITECTURE.md
+++ b/orchestration/sim/cluster0/ARCHITECTURE.md
@@ -0,0 +1,77 @@
+# cast01–cast03 lab architecture
+
+## Goals
+
+Simulate production patterns on three isolated Alpine VMs with **shared config only** on NFS (`/shared/cluster/`). No database files on NFS — each node owns `/var/lib/mysql` locally.
+
+## Topology
+
+```text
+                    ┌─────────────────────────────────────┐
+                    │  FE 10.7.0.10 (NFS export /shared)  │
+                    └──────────────────┬──────────────────┘
+                                       │ NFS (configs/scripts only)
+         ┌─────────────────────────────┼─────────────────────────────┐
+         ▼                             ▼                             ▼
+   ┌───────────┐               ┌───────────┐               ┌───────────┐
+   │  cast01   │  async GTID   │  cast02   │  async GTID   │  cast03   │
+   │ .16.236   │◄──────────────│ .16.237   │◄──────────────│ .16.238   │
+   │ PRIMARY   │   replica     │ REPLICA   │   replica     │ REPLICA   │
+   │ MariaDB   │               │ MariaDB   │               │ MariaDB   │
+   │ nginx/php │               │ nginx/php │               │ nginx/php │
+   └───────────┘               └───────────┘               └───────────┘
+         ▲                             │                             │
+         └──────── writes + default reads ────────────────────────────┘
+                    (replicas: read-only, failover drill)
+```
+
+| Node | IP | MariaDB | App role (planned) |
+|------|-----|---------|-------------------|
+| cast01 | 10.7.16.236 | **Primary** (read/write) | Crashes console primary target |
+| cast02 | 10.7.16.237 | Replica (read_only) | App worker / read replica |
+| cast03 | 10.7.16.238 | Replica (read_only) | App worker / read replica |
+
+## MariaDB
+
+- **Engine:** MariaDB 11.8 (Alpine 3.24 packages — no edge required)
+- **Replication:** GTID async primary → two replicas (`server-id` 1/2/3)
+- **Bootstrap:** app DB dump over TCP (`--skip-ssl`); GTID position copied from primary; `replicate-do-db` filters ongoing binlog to app schemas only (avoids `mysql.help_topic` dump failures)
+- **Databases:** `androidcast_crashes`, `url_shortener` (schemas from project SQL in `/shared/cluster/sql/`)
+- **Credentials:** see `credentials.txt` (same lab password on all nodes)
+
+## PHP
+
+- **Version:** 8.3 (`php83-fpm`) on all nodes — matches modern Alpine; prod BE remains 8.1 until upgraded separately
+- **Socket:** `127.0.0.1:9000`
+
+## DNS
+
+Pending external records. Until then: `/etc/hosts` block maintained by `baseline.sh`.
+
+## Service split (selected for lab)
+
+| Service | Write DB host | Read DB host (lab default) | Future app path (local disk) |
+|---------|---------------|----------------------------|------------------------------|
+| Crashes / tickets / RA | cast01 | cast01 (cast02/03 for drill) | `/var/www/androidcast/` per node |
+| URL shortener API | cast01 | cast01 | `/var/www/url-shortener/` on cast03 |
+| Graphs ingest | cast01 | cast01 | shared crashes backend |
+
+App deploys stay on **local VM disk**, not NFS.
+
+## Operations
+
+```sh
+sudo sh /shared/cluster/scripts/baseline.sh          # per-node baseline
+sudo sh /shared/cluster/scripts/mariadb-primary.sh   # cast01 only
+sudo sh /shared/cluster/scripts/load-schemas.sh      # cast01 only
+sudo sh /shared/cluster/scripts/mariadb-replica.sh # cast02 + cast03
+sudo sh /shared/cluster/scripts/verify-cluster.sh  # any node
+```
+
+NFS mount may be `noexec` — always invoke scripts with `sh`.
+
+## Not in scope yet
+
+- ProxySQL / MaxScale (add if read splitting becomes necessary)
+- Galera multi-primary (overkill for 3-node lab; async matches prod BE migration path)
+- TLS between nodes (intra VLAN)
--- a/orchestration/sim/cluster0/DEPLOY.md
+++ b/orchestration/sim/cluster0/DEPLOY.md
@@ -0,0 +1,108 @@
+# cast cluster — unattended populate & verify
+
+This directory is the **single source of truth** for lab/staging cluster deployment.
+Store scripts, configs, SQL, and credentials here only — **never** MariaDB datadirs or dumps.
+
+## Fresh cluster workflow (cluster0 → clusterN)
+
+### 0. FE: create NFS export + seed this tree
+
+Copy or rsync this entire `cluster/` tree to the FE export, e.g.:
+
+`10.7.0.10:/mnt/raid0/xendomains/domU_cast_cluster_0/shared/cluster/`
+
+Edit `cluster.env` for the new cluster (IPs, export path, `CLUSTER_NAME`, `PUBLIC_ORIGIN`).
+
+### 1. Create 3 empty Alpine VMs
+
+Each VM needs hostname `cast01` / `cast02` / `cast03` and static IP from `cluster.env`.
+
+Copy bootstrap env once (before NFS is mounted):
+
+```sh
+sudo install -m 644 bootstrap/cluster0-bootstrap.env /etc/cast-cluster.env
+```
+
+### 2. Populate (unattended)
+
+**Option A — coordinator** (from a host with passwordless SSH to all cast nodes — typically your workstation, not cast01):
+
+```sh
+ssh ai@cast01  # or run from laptop with SSH config for cast01–03
+sudo sh /shared/cluster/populate_lab_setup.sh --coordinator
+```
+
+If inter-node SSH is not configured, run **Option B** on each VM instead; cluster verify uses HTTP checks to peer IPs.
+
+**Option B — per node** (parallel OK; replicas wait for primary):
+
+```sh
+sudo sh /shared/cluster/populate_lab_setup.sh
+```
+
+Phases:
+
+| Phase | What |
+|-------|------|
+| `baseline` | NFS, hosts, `apk upgrade`, nginx/php83, packages |
+| `db` | cast01: GTID primary + schemas; cast02/03: replicas |
+| `app` | git/bundle deploy, nginx vhost, lab `config.php` |
+| `all` | baseline → db → app → `verify_lab_setup.sh --local` |
+
+Always invoke with `sh` — NFS may be mounted `noexec`.
+
+### 3. DNS + FE redirect
+
+Update external DNS and FE nginx upstream to point at the new cluster (`FE_PROXY_TARGET` in `cluster.env`).
+
+### 4. Global verify
+
+```sh
+sudo sh /shared/cluster/verify_global_setup.sh
+```
+
+Checks `PUBLIC_ORIGIN` paths (crashes UI, assets, diag, short links).
+
+## Staging / cloud
+
+Same scripts; change only `cluster.env`:
+
+- `FE_NFS_*` → object storage or config bucket mount
+- `APP_SOURCE=bundle` + `release/current/` tarball from CI
+- `PUBLIC_ORIGIN` → staging URL
+- IPs / hostnames → cloud private network
+
+## Files
+
+| File | Purpose |
+|------|---------|
+| `cluster.env` | Cluster-specific variables (IPs, NFS, URLs, git) |
+| `credentials.txt` | Shared lab passwords |
+| `populate_lab_setup.sh` | Main unattended installer |
+| `verify_lab_setup.sh` | Internal cluster health |
+| `verify_global_setup.sh` | Post-DNS public checks |
+| `scripts/` | Baseline, MariaDB, deploy, nginx |
+| `sql/` | Schema + migrations |
+| `nginx/apps-port80.conf` | php83-fpm TCP vhost |
+| `bootstrap/` | Minimal `/etc/cast-cluster.env` for first boot |
+
+See `ARCHITECTURE.md` for topology.
+
+## Journal (read from FE if VMs fail)
+
+Every populate/verify step appends to **`/shared/journal/{CLUSTER_NAME}.journal.tsv`** with status **OK**, **NOK**, **WARN**, or **START**.
+
+Human summary (regenerated automatically):
+
+```sh
+less /shared/journal/cluster0.summary.md
+grep NOK /shared/journal/cluster0.journal.tsv
+```
+
+On FE host:
+
+```sh
+less /mnt/raid0/xendomains/domU_cast_cluster_0/shared/journal/cluster0.summary.md
+```
+
+See `/shared/journal/README.md`. VM disk snapshots are out-of-band (ask sysop) — not stored on NFS.
--- a/orchestration/sim/cluster0/README.md
+++ b/orchestration/sim/cluster0/README.md
@@ -0,0 +1,17 @@
+# cast01–cast03 lab cluster
+
+**Runbook:** [DEPLOY.md](DEPLOY.md) — unattended populate & verify.
+
+Quick start on a fresh node:
+
+```sh
+sudo install -m 644 bootstrap/cluster0-bootstrap.env /etc/cast-cluster.env   # once, before NFS
+sudo sh /shared/cluster/populate_lab_setup.sh                                 # or --coordinator
+sudo sh /shared/cluster/verify_lab_setup.sh --local
+# after DNS:
+sudo sh /shared/cluster/verify_global_setup.sh
+```
+
+See `ARCHITECTURE.md` for topology and `cluster.env` for cluster-specific settings.
+
+**Journal:** every setup action is logged under `/shared/journal/` with **OK** / **NOK** — readable from FE if VMs are down. See `/shared/journal/README.md`.
--- a/orchestration/sim/cluster0/bootstrap/cluster0-bootstrap.env
+++ b/orchestration/sim/cluster0/bootstrap/cluster0-bootstrap.env
@@ -0,0 +1,5 @@
+# Copy to /etc/cast-cluster.env on each fresh VM before first populate run.
+# Only needed to mount NFS when /shared/cluster/cluster.env is not reachable yet.
+FE_NFS_SERVER=10.7.0.10
+FE_NFS_EXPORT_PATH=/mnt/raid0/xendomains/domU_cast_cluster_0/shared
+SHARED_MOUNT=/shared
--- a/orchestration/sim/cluster0/cluster.env
+++ b/orchestration/sim/cluster0/cluster.env
@@ -0,0 +1,42 @@
+# cluster0 — cast01–cast03 lab (edit for new clusters: cluster1, staging, etc.)
+CLUSTER_NAME=cluster0
+CLUSTER_ID=0
+
+FE_NFS_SERVER=10.7.0.10
+FE_NFS_EXPORT_PATH=/mnt/raid0/xendomains/domU_cast_cluster_0/shared
+SHARED_MOUNT=/shared
+
+CAST01_HOST=cast01
+CAST01_IP=10.7.16.236
+CAST02_HOST=cast02
+CAST02_IP=10.7.16.237
+CAST03_HOST=cast03
+CAST03_IP=10.7.16.238
+CAST_DOMAIN=intra.raptor.org
+
+PRIMARY_DB_HOST=cast01
+PRIMARY_DB_PORT=3306
+
+ARTC0_HOST=artc0
+ARTC0_IP=10.7.16.128
+
+SSH_USER=ai
+
+GIT_ORIGIN=git://f0xx.org/android_cast
+GIT_BRANCH=next
+# git | bundle — bundle uses SHARED_MOUNT/cluster/release/current/
+APP_SOURCE=git
+
+APP_ROOT=/var/www/localhost/htdocs/apps/app/androidcast_project
+APP_BASE_PATH=/app/androidcast_project/crashes
+APP_HUB_PATH=/app/androidcast_project
+
+# Set after DNS + FE redirect, then run verify_global_setup.sh
+PUBLIC_ORIGIN=https://apps.f0xx.org
+SHORT_LINKS_PUBLIC_BASE=https://s.f0xx.org
+FE_PROXY_TARGET=cast01.intra.raptor.org:80
+
+EXPECTED_CRASHES_TABLES=16
+EXPECTED_URL_SHORTENER_TABLES=4
+
+APK_PACKAGES_BASE="nginx php83 php83-fpm php83-cli php83-session php83-pdo php83-pdo_mysql php83-pdo_sqlite php83-sqlite3 php83-mbstring php83-json php83-curl php83-openssl php83-xml php83-zip php83-phar php83-opcache mariadb-client git rsync curl bash ca-certificates sqlite wireguard-tools nfs-utils openssh-client"
--- a/orchestration/sim/cluster0/credentials.txt
+++ b/orchestration/sim/cluster0/credentials.txt
@@ -0,0 +1,22 @@
+# cast cluster lab credentials (NOT production — shared intentionally for lab safety)
+# Same values on cast01, cast02, cast03 unless noted.
+
+[mariadb_root]
+# Alpine default: unix socket auth as root, no password
+
+[mariadb_app]
+user=androidcast
+password=cast-cluster-dev
+host_write=cast01
+host_read=cast01
+port=3306
+databases=androidcast_crashes,url_shortener
+
+[mariadb_replication]
+user=repl
+password=cast-cluster-dev
+allowed_from=10.7.16.%
+
+[ssh]
+user=ai
+sudo=passwordless
--- a/orchestration/sim/cluster0/journal/README.md
+++ b/orchestration/sim/cluster0/journal/README.md
@@ -0,0 +1,55 @@
+# Cluster setup journal (`/shared/journal/`)
+
+Human-readable audit trail for populate / verify runs. **Read from FE** when VMs are down or after reboot failure — no SSH to cast nodes required.
+
+## Files (per cluster)
+
+| File | Purpose |
+|------|---------|
+| `{CLUSTER_NAME}.journal.tsv` | Append-only machine log (tab-separated) |
+| `{CLUSTER_NAME}.summary.md` | Regenerated table with **OK** / **NOK** / **WARN** |
+| `README.md` | This file |
+
+Example for cluster0:
+
+```
+/shared/journal/cluster0.journal.tsv
+/shared/journal/cluster0.summary.md
+```
+
+## TSV columns
+
+```
+timestamp_utc  host  actor  phase  action  status  detail
+```
+
+**status:** `START` → work begun; `OK` / `NOK` / `WARN` → outcome.
+
+**actor:** script name (`populate`, `verify_lab`, `verify_global`, `baseline`, …).
+
+## Who writes entries
+
+All cluster scripts call `journal_*` helpers in `scripts/lib/common.sh`:
+
+- `populate_lab_setup.sh` — each phase and sub-script
+- `verify_lab_setup.sh` / `verify_global_setup.sh` — pass/fail
+- `run_script` — every `scripts/*.sh` invocation
+
+## Read from FE (10.7.0.10)
+
+```sh
+less /mnt/raid0/xendomains/domU_cast_cluster_0/shared/journal/cluster0.summary.md
+grep NOK /mnt/raid0/xendomains/domU_cast_cluster_0/shared/journal/cluster0.journal.tsv
+```
+
+## Rebuild summary manually
+
+```sh
+sudo sh /shared/cluster/scripts/journal-rebuild-summary.sh
+```
+
+## Snapshots
+
+If reboot breaks a node and journal shows last **OK** before power-loss, ask sysop for **cluster snapshot restore** (XEN/HVM snapshot — out of band, not stored on NFS).
+
+Do **not** store VM disk images or MariaDB datadirs in `/shared/`.
--- a/orchestration/sim/cluster0/mariadb/primary.cnf
+++ b/orchestration/sim/cluster0/mariadb/primary.cnf
@@ -0,0 +1,10 @@
+[mysqld]
+# cast01 — MariaDB primary (GTID async replication)
+server-id=1
+log_bin=mysql-bin
+binlog_format=ROW
+gtid_domain_id=1
+log_slave_updates=1
+expire_logs_days=7
+bind-address=0.0.0.0
+read_only=0
--- a/orchestration/sim/cluster0/mariadb/replica.cnf
+++ b/orchestration/sim/cluster0/mariadb/replica.cnf
@@ -0,0 +1,11 @@
+[mysqld]
+# cast02/cast03 — MariaDB replica (server-id set by mariadb-replica.sh)
+log_bin=mysql-bin
+binlog_format=ROW
+gtid_domain_id=1
+log_slave_updates=1
+bind-address=0.0.0.0
+read_only=1
+relay_log=relay-bin
+replicate-do-db=androidcast_crashes
+replicate-do-db=url_shortener
--- a/orchestration/sim/cluster0/nginx/apps-port80.conf
+++ b/orchestration/sim/cluster0/nginx/apps-port80.conf
@@ -0,0 +1,119 @@
+# Full androidcast vhost for Alpine BE — listen 80 only.
+# FE (apps.f0xx.org) proxies here: proxy_pass http://artc0.intra.raptor.org:80;
+# Port 8089 is a different service (e.g. Janus) — not this vhost.
+
+server {
+	listen 80 default_server;
+	listen [::]:80 default_server;
+
+	# OTA v0 static tree — devices fetch https://apps.f0xx.org/v0/ota/channel/stable.json
+	# Populate: builder auto_deploy, or rsync out/ota/v0/ → .../ota-artifacts/v0/
+	location ^~ /v0/ota/ {
+		alias /var/www/localhost/htdocs/apps/app/androidcast_project/ota-artifacts/v0/ota/;
+		add_header Cache-Control "public, max-age=60";
+	}
+
+	location = /app/androidcast_project {
+		return 301 /app/androidcast_project/;
+	}
+
+	location = /app/androidcast_project/index.php {
+		include fastcgi_params;
+		fastcgi_pass 127.0.0.1:9000;
+		fastcgi_param SCRIPT_FILENAME /var/www/localhost/htdocs/apps/app/androidcast_project/index.php;
+		fastcgi_param SCRIPT_NAME /app/androidcast_project/index.php;
+	}
+
+	location = /app/androidcast_project/ {
+		rewrite ^ /app/androidcast_project/index.php last;
+	}
+
+	location /app/androidcast_project/ {
+		alias /var/www/localhost/htdocs/apps/app/androidcast_project/;
+		index index.php index.html;
+		try_files $uri $uri/ =404;
+	}
+
+	location = /app/androidcast_project/crashes {
+		return 301 /app/androidcast_project/crashes/;
+	}
+
+	location = /app/androidcast_project/graphs {
+		return 301 /app/androidcast_project/graphs/;
+	}
+
+	location ^~ /app/androidcast_project/crashes/assets/ {
+		alias /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/crash_reporter/backend/public/assets/;
+	}
+
+	location ^~ /app/androidcast_project/graphs/assets/ {
+		alias /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/crash_reporter/backend/public/assets/;
+	}
+
+	location = /app/androidcast_project/crashes/api/upload.php {
+		include fastcgi_params;
+		fastcgi_pass 127.0.0.1:9000;
+		fastcgi_param SCRIPT_FILENAME /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/crash_reporter/backend/public/api/upload.php;
+		fastcgi_param SCRIPT_NAME /app/androidcast_project/crashes/api/upload.php;
+		fastcgi_param REQUEST_URI $request_uri;
+		client_max_body_size 4m;
+	}
+
+	location ^~ /app/androidcast_project/crashes/ {
+		include fastcgi_params;
+		fastcgi_pass 127.0.0.1:9000;
+		fastcgi_param SCRIPT_FILENAME /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/crash_reporter/backend/public/index.php;
+		fastcgi_param SCRIPT_NAME /app/androidcast_project/crashes/index.php;
+		fastcgi_param REQUEST_URI $request_uri;
+		client_max_body_size 4m;
+	}
+
+	location = /app/androidcast_project/graphs/api/graphs.php {
+		include fastcgi_params;
+		fastcgi_pass 127.0.0.1:9000;
+		fastcgi_param SCRIPT_FILENAME /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/crash_reporter/backend/public/api/graphs.php;
+		fastcgi_param SCRIPT_NAME /app/androidcast_project/graphs/api/graphs.php;
+		fastcgi_param REQUEST_URI $request_uri;
+	}
+
+	location = /app/androidcast_project/graphs/api/graph_upload.php {
+		include fastcgi_params;
+		fastcgi_pass 127.0.0.1:9000;
+		fastcgi_param SCRIPT_FILENAME /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/crash_reporter/backend/public/api/graph_upload.php;
+		fastcgi_param SCRIPT_NAME /app/androidcast_project/graphs/api/graph_upload.php;
+		fastcgi_param REQUEST_URI $request_uri;
+		client_max_body_size 4m;
+	}
+
+	location ^~ /app/androidcast_project/graphs/ {
+		include fastcgi_params;
+		fastcgi_pass 127.0.0.1:9000;
+		fastcgi_param SCRIPT_FILENAME /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/crash_reporter/backend/public/index.php;
+		fastcgi_param SCRIPT_NAME /app/androidcast_project/graphs/index.php;
+		fastcgi_param REQUEST_URI $request_uri;
+		client_max_body_size 4m;
+	}
+
+	location = /app/androidcast_project/build {
+		return 301 /app/androidcast_project/build/;
+	}
+
+	location ^~ /app/androidcast_project/build/assets/ {
+		alias /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/build_console/backend/public/assets/;
+	}
+
+	location ^~ /app/androidcast_project/build/ {
+		include fastcgi_params;
+		fastcgi_pass 127.0.0.1:9000;
+		fastcgi_param SCRIPT_FILENAME /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/build_console/backend/public/index.php;
+		fastcgi_param SCRIPT_NAME /app/androidcast_project/build/index.php;
+		fastcgi_param REQUEST_URI $request_uri;
+		client_max_body_size 512m;
+		fastcgi_read_timeout 1800s;
+	}
+
+	location ~ \.php$ {
+		include /etc/nginx/fastcgi.conf;
+		fastcgi_pass 127.0.0.1:9000;
+	}
+}
--- a/orchestration/sim/cluster0/populate_lab_setup.sh
+++ b/orchestration/sim/cluster0/populate_lab_setup.sh
@@ -0,0 +1,136 @@
+#!/bin/sh
+# Unattended cast cluster population — dev lab and staging template.
+# Usage:
+#   sudo sh populate_lab_setup.sh              # full setup on this node
+#   sudo sh populate_lab_setup.sh --coordinator # from jump host: all nodes in order
+#   sudo sh populate_lab_setup.sh --phase baseline|db|app|all
+set -eu
+
+ROOT="$(cd "$(dirname "$0")" && pwd)"
+export CAST_CLUSTER_ROOT="$ROOT"
+# shellcheck source=/dev/null
+. "$ROOT/scripts/lib/common.sh"
+export JOURNAL_ACTOR=populate
+
+PHASE=all
+COORDINATOR=0
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --coordinator) COORDINATOR=1; shift ;;
+    --phase) PHASE="${2:-all}"; shift 2 ;;
+    --help|-h)
+      echo "Usage: sudo sh populate_lab_setup.sh [--coordinator] [--phase baseline|db|app|all]"
+      exit 0
+      ;;
+    *) die "unknown arg: $1" ;;
+  esac
+done
+
+run_phase_baseline() {
+  JOURNAL_PHASE=baseline
+  journal_start phase_baseline "$(host_short)"
+  log "phase baseline on $(host_short)"
+  journal_start apk_upgrade
+  apk update
+  apk upgrade -U -a || apk upgrade -U || true
+  journal_ok apk_upgrade
+  run_script "$ROOT/scripts/baseline.sh"
+  journal_ok phase_baseline
+}
+
+run_phase_db() {
+  JOURNAL_PHASE=db
+  journal_start phase_db "$(host_short)"
+  if is_primary_node; then
+    log "phase db primary on $(host_short)"
+    run_script "$ROOT/scripts/mariadb-primary.sh"
+    run_script "$ROOT/scripts/load-schemas.sh"
+    journal_ok phase_db "primary"
+    return
+  fi
+  if is_replica_node; then
+    log "phase db replica on $(host_short) — waiting for primary"
+    wait_for_primary_db
+    wait_for_tcp "$PRIMARY_DB_HOST" "$PRIMARY_DB_PORT" 300
+    run_script "$ROOT/scripts/mariadb-replica.sh"
+    journal_ok phase_db "replica"
+    return
+  fi
+  journal_nok phase_db "unknown host"
+  die "unknown host for db phase: $(host_short)"
+}
+
+run_phase_app() {
+  JOURNAL_PHASE=app
+  journal_start phase_app "$(host_short)"
+  log "phase app on $(host_short)"
+  if is_replica_node; then
+    wait_for_primary_db
+  fi
+  run_script "$ROOT/scripts/deploy-app.sh"
+  run_script "$ROOT/scripts/configure-nginx.sh"
+  rc-service php-fpm83 restart 2>/dev/null || true
+  rc-service nginx reload 2>/dev/null || rc-service nginx restart 2>/dev/null || true
+  if is_primary_node; then
+    rc-service mariadb restart 2>/dev/null || true
+  elif is_replica_node; then
+    rc-service mariadb restart 2>/dev/null || true
+  fi
+  journal_ok phase_app
+}
+
+run_local() {
+  ensure_shared_mounted
+  journal_start populate_local "phase=${PHASE}"
+  case "$PHASE" in
+    baseline) run_phase_baseline ;;
+    db) run_phase_baseline; run_phase_db ;;
+    app) run_phase_app ;;
+    all)
+      run_phase_baseline || exit 1
+      run_phase_db || exit 1
+      run_phase_app || exit 1
+      run_script "$ROOT/verify_lab_setup.sh" --local || exit 1
+      ;;
+    *) journal_nok populate_local "unknown phase"; die "unknown phase: $PHASE" ;;
+  esac
+  journal_ok populate_local "phase=${PHASE}"
+}
+
+run_coordinator() {
+  ensure_shared_mounted
+  journal_start populate_coordinator "nodes=${ALL_CAST_HOSTS}"
+  log "coordinator: ${CLUSTER_NAME} nodes=${ALL_CAST_HOSTS}"
+
+  for H in $CAST01_HOST $CAST02_HOST $CAST03_HOST; do
+    ssh_node "$H" "sudo sh ${SHARED_MOUNT}/cluster/populate_lab_setup.sh --phase baseline" &
+  done
+  wait || die "baseline failed on one or more nodes"
+
+  ssh_node "$CAST01_HOST" "sudo sh ${SHARED_MOUNT}/cluster/populate_lab_setup.sh --phase db"
+
+  for H in $CAST02_HOST $CAST03_HOST; do
+    ssh_node "$H" "sudo sh ${SHARED_MOUNT}/cluster/populate_lab_setup.sh --phase db" &
+  done
+  wait
+
+  for H in $CAST01_HOST $CAST02_HOST $CAST03_HOST; do
+    ssh_node "$H" "sudo sh ${SHARED_MOUNT}/cluster/populate_lab_setup.sh --phase app" &
+  done
+  wait
+
+  sh "$ROOT/verify_lab_setup.sh" --cluster
+  journal_ok populate_coordinator
+  log "populate_coordinator_ok ${CLUSTER_NAME}"
+}
+
+if [ "$(id -u)" -ne 0 ]; then
+  die "run as root (sudo sh populate_lab_setup.sh)"
+fi
+
+if [ "$COORDINATOR" -eq 1 ]; then
+  run_coordinator
+else
+  run_local
+fi
--- a/orchestration/sim/cluster0/scripts/baseline.sh
+++ b/orchestration/sim/cluster0/scripts/baseline.sh
@@ -0,0 +1,67 @@
+#!/bin/sh
+# cast cluster baseline — packages, NFS, hosts, nginx/php smoke vhost.
+set -eu
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+# shellcheck source=/dev/null
+. "$ROOT/scripts/lib/common.sh"
+ensure_shared_mounted
+
+HOST="$(host_short)"
+STAGE="$(mktemp -d /var/tmp/cast-cluster.XXXXXX)"
+trap 'rm -rf "$STAGE"' EXIT
+
+log "baseline start on $HOST"
+
+grep -v "${FE_NFS_EXPORT_PATH}" /etc/fstab > "$STAGE/fstab.new" || true
+echo "$FSTAB_NFS_LINE" >> "$STAGE/fstab.new"
+mv "$STAGE/fstab.new" /etc/fstab
+mkdir -p "$SHARED_MOUNT"
+rc-update add nfsmount boot 2>/dev/null || true
+mount "$SHARED_MOUNT" 2>/dev/null || mount -a 2>/dev/null || true
+
+grep -v '# cast-cluster' /etc/hosts | grep -v 'cast0[123]\.' | grep -v 'artc0\.' > "$STAGE/hosts.new" || cp /etc/hosts "$STAGE/hosts.new"
+cat >> "$STAGE/hosts.new" <<EOF
+# cast-cluster ${CLUSTER_NAME}
+${CAST01_IP} ${CAST01_HOST} ${CAST01_HOST}.${CAST_DOMAIN}
+${CAST02_IP} ${CAST02_HOST} ${CAST02_HOST}.${CAST_DOMAIN}
+${CAST03_IP} ${CAST03_HOST} ${CAST03_HOST}.${CAST_DOMAIN}
+${ARTC0_IP} ${ARTC0_HOST} ${ARTC0_HOST}.${CAST_DOMAIN} alpine-be
+EOF
+mv "$STAGE/hosts.new" /etc/hosts
+
+apk update
+PKGS="$APK_PACKAGES_BASE"
+if is_primary_node; then
+  PKGS="$PKGS mariadb"
+fi
+# shellcheck disable=SC2086
+apk add --no-cache $PKGS
+
+rc-update add nginx default 2>/dev/null || true
+rc-update add php-fpm83 default 2>/dev/null || true
+run_script "$ROOT/scripts/configure-php-fpm.sh"
+rc-service php-fpm83 restart 2>/dev/null || rc-service php-fpm83 start 2>/dev/null || true
+rc-service nginx start 2>/dev/null || true
+
+if is_primary_node; then
+  grep -q '^skip-networking' /etc/my.cnf.d/mariadb-server.cnf 2>/dev/null && \
+    sed -i 's/^skip-networking/#skip-networking/' /etc/my.cnf.d/mariadb-server.cnf || true
+  grep -q '^bind-address' /etc/my.cnf.d/mariadb-server.cnf 2>/dev/null || \
+    printf '\n[mysqld]\nbind-address = 0.0.0.0\n' >> /etc/my.cnf.d/mariadb-server.cnf
+  rc-update add mariadb default 2>/dev/null || true
+  rc-service mariadb restart 2>/dev/null || rc-service mariadb start 2>/dev/null || true
+fi
+
+mkdir -p /var/www/localhost/htdocs
+echo "<html><body><h1>${HOST}</h1><p>${CLUSTER_NAME} node</p></body></html>" > /var/www/localhost/htdocs/index.html
+chown -R nginx:nginx /var/www/localhost/htdocs 2>/dev/null || true
+
+mkdir -p "$ROOT"
+IP="$(ip -4 -o addr show eth0 2>/dev/null | awk '{print $4}' | cut -d/ -f1)"
+echo "$HOST $(date -Iseconds) ip=$IP cluster=$CLUSTER_NAME" >> "$ROOT/seen.log"
+printf '{"host":"%s","ip":"%s","cluster":"%s","configured_at":"%s"}\n' \
+  "$HOST" "$IP" "$CLUSTER_NAME" "$(date -Iseconds)" > "$ROOT/${HOST}.json"
+
+mount | grep -q " on ${SHARED_MOUNT} " || die "NFS not mounted"
+curl -sS -o /dev/null -w "nginx_http=%{http_code}\n" http://127.0.0.1/
+log "baseline_ok $HOST"
--- a/orchestration/sim/cluster0/scripts/configure-nginx.sh
+++ b/orchestration/sim/cluster0/scripts/configure-nginx.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+set -eu
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+# shellcheck source=/dev/null
+. "$ROOT/scripts/lib/common.sh"
+ensure_shared_mounted
+
+install -D -m 644 "$ROOT/nginx/apps-port80.conf" /etc/nginx/http.d/androidcast.conf
+rm -f /etc/nginx/http.d/default.conf /etc/nginx/http.d/cluster.conf 2>/dev/null || true
+nginx -t
+rc-service nginx reload
+log "nginx androidcast vhost installed"
--- a/orchestration/sim/cluster0/scripts/configure-php-fpm.sh
+++ b/orchestration/sim/cluster0/scripts/configure-php-fpm.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+set -eu
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+# shellcheck source=/dev/null
+. "$ROOT/scripts/lib/common.sh"
+ensure_shared_mounted
+
+CONF=/etc/php83/php-fpm.d/www.conf
+grep -q '^listen = 127.0.0.1:9000' "$CONF" 2>/dev/null || \
+  sed -i 's|^listen = .*|listen = 127.0.0.1:9000|' "$CONF"
+grep -q '^listen.allowed_clients' "$CONF" 2>/dev/null || \
+  printf '\nlisten.allowed_clients = 127.0.0.1\n' >> "$CONF"
+log "php-fpm83 listen 127.0.0.1:9000"
--- a/orchestration/sim/cluster0/scripts/deploy-app.sh
+++ b/orchestration/sim/cluster0/scripts/deploy-app.sh
@@ -0,0 +1,109 @@
+#!/bin/sh
+# Deploy androidcast project tree + lab config.php (local disk, not NFS).
+set -eu
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+# shellcheck source=/dev/null
+. "$ROOT/scripts/lib/common.sh"
+ensure_shared_mounted
+
+APP_PASS="$(read_cred mariadb_app password)"
+DB_HOST="$PRIMARY_DB_HOST"
+if is_primary_node; then
+  DB_HOST=127.0.0.1
+fi
+
+mkdir -p "$(dirname "$APP_ROOT")"
+GIT_ROOT="${APP_ROOT}/android_cast"
+BACKEND="${GIT_ROOT}/examples/crash_reporter/backend"
+CONFIG="${BACKEND}/config/config.php"
+
+case "$APP_SOURCE" in
+  git)
+    if [ -d "${GIT_ROOT}/.git" ]; then
+      log "git pull ${GIT_BRANCH} in ${GIT_ROOT}"
+      git config --global --add safe.directory "$GIT_ROOT" 2>/dev/null || true
+      git -C "$GIT_ROOT" fetch origin
+      git -C "$GIT_ROOT" checkout "$GIT_BRANCH"
+      git -C "$GIT_ROOT" pull --ff-only origin "$GIT_BRANCH"
+    else
+      log "git clone ${GIT_ORIGIN} → ${GIT_ROOT}"
+      rm -rf "$GIT_ROOT"
+      git clone --branch "$GIT_BRANCH" --depth 1 "$GIT_ORIGIN" "$GIT_ROOT"
+      git config --global --add safe.directory "$GIT_ROOT" 2>/dev/null || true
+    fi
+    ;;
+  bundle)
+    BUNDLE="${ROOT}/release/current"
+    [ -d "$BUNDLE" ] || die "missing bundle dir $BUNDLE (APP_SOURCE=bundle)"
+    log "rsync bundle $BUNDLE → ${APP_ROOT}"
+    mkdir -p "$APP_ROOT"
+    rsync -a --delete "$BUNDLE/" "$APP_ROOT/"
+    ;;
+  *)
+    die "unknown APP_SOURCE=$APP_SOURCE"
+    ;;
+esac
+
+mkdir -p "${BACKEND}/config" "${BACKEND}/data" "${BACKEND}/storage"
+chown -R nginx:nginx "$APP_ROOT" 2>/dev/null || true
+
+if [ ! -f "$CONFIG" ] || [ "${FORCE_CONFIG:-1}" = "1" ]; then
+  log "writing lab config.php"
+  cat > "$CONFIG" <<PHP
+<?php
+return [
+    'app_name' => 'Android Cast Issues',
+    'base_path' => '${APP_BASE_PATH}',
+    'db' => [
+        'driver' => 'mariadb',
+        'sqlite_path' => __DIR__ . '/../data/crashes.sqlite',
+        'mysql' => [
+            'host' => '${DB_HOST}',
+            'port' => ${PRIMARY_DB_PORT},
+            'socket' => '',
+            'database' => 'androidcast_crashes',
+            'username' => 'androidcast',
+            'password' => '${APP_PASS}',
+            'charset' => 'utf8mb4',
+        ],
+    ],
+    'session_name' => 'ac_crash_sess',
+    'session_cookie_path' => '${APP_HUB_PATH}',
+    'debug' => true,
+    'rbac' => [
+        'default_company_slug' => 'default',
+        'default_company_id' => 1,
+    ],
+    'analytics' => ['enabled' => false, 'measurement_id' => '', 'debug' => false],
+    'remote_access' => [
+        'wg_endpoint' => '',
+        'wg_server_public_key' => '',
+        'provision_peers' => false,
+        'require_wg_tools' => false,
+    ],
+    'auth' => [
+        'encryption_key' => 'lab-cluster-dev-32-char-min-secret!!',
+        'max_attempts' => 8,
+        'lockout_window_minutes' => 15,
+    ],
+    'url_shortener' => [
+        'enabled' => true,
+        'public_base' => '${SHORT_LINKS_PUBLIC_BASE}',
+        'db' => [
+            'mysql' => [
+                'host' => '${DB_HOST}',
+                'port' => ${PRIMARY_DB_PORT},
+                'socket' => '',
+                'database' => 'url_shortener',
+                'username' => 'androidcast',
+                'password' => '${APP_PASS}',
+                'charset' => 'utf8mb4',
+            ],
+        ],
+    ],
+];
+PHP
+  chown nginx:nginx "$CONFIG" 2>/dev/null || true
+fi
+
+log "deploy-app_ok $(host_short)"
--- a/orchestration/sim/cluster0/scripts/journal-backfill-cluster0.sh
+++ b/orchestration/sim/cluster0/scripts/journal-backfill-cluster0.sh
@@ -0,0 +1,57 @@
+#!/bin/sh
+# One-time backfill of cluster0 journal from known-good state (2026-06-13 session).
+# Safe to re-run: appends only if journal is empty.
+set -eu
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+# shellcheck source=/dev/null
+. "$ROOT/scripts/lib/common.sh"
+ensure_shared_mounted
+journal_init
+
+if [ -f "$JOURNAL_TSV" ] && [ "$(wc -l < "$JOURNAL_TSV")" -gt 1 ]; then
+  log "journal already has entries — skip backfill (see $JOURNAL_TSV)"
+  sh "$ROOT/scripts/journal-rebuild-summary.sh"
+  exit 0
+fi
+
+export JOURNAL_ACTOR=backfill
+TS="2026-06-13T12:00:00+00:00"
+
+append() {
+  _host="$1"
+  _phase="$2"
+  _action="$3"
+  _status="$4"
+  _detail="$5"
+  printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
+    "$TS" "$_host" "$JOURNAL_ACTOR" "$_phase" "$_action" "$_status" "$_detail" >> "$JOURNAL_TSV"
+}
+
+# Infrastructure (all nodes)
+for H in cast01 cast02 cast03; do
+  append "$H" baseline nfs_mount OK "/shared from FE export"
+  append "$H" baseline packages OK "nginx php83-fpm mariadb-client nfs-utils"
+  append "$H" baseline hosts OK "cast-cluster /etc/hosts block"
+done
+
+# MariaDB
+append cast01 db mariadb-primary OK "GTID primary server-id=1"
+append cast01 db load-schemas OK "16 crashes + 4 url_shortener tables"
+append cast02 db mariadb-replica OK "GTID replica IO+SQL running"
+append cast03 db mariadb-replica OK "GTID replica IO+SQL running"
+
+# App + nginx
+for H in cast01 cast02 cast03; do
+  append "$H" app deploy-app OK "git next → android_cast/"
+  append "$H" app configure-nginx OK "apps-port80.conf php83:9000"
+done
+
+# Verification
+append cast01 verify verify_lab OK "mode=local"
+append cast02 verify verify_lab OK "mode=local"
+append cast03 verify verify_lab OK "mode=local"
+append cast01 verify verify_lab OK "mode=cluster HTTP peers"
+append cast01 verify verify_global START "pending DNS/FE cutover"
+
+sh "$ROOT/scripts/journal-rebuild-summary.sh"
+log "backfill_ok → $JOURNAL_TSV"
--- a/orchestration/sim/cluster0/scripts/journal-rebuild-summary.sh
+++ b/orchestration/sim/cluster0/scripts/journal-rebuild-summary.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+# Regenerate human-readable OK/NOK summary from journal TSV (safe on FE via NFS read).
+set -eu
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+# shellcheck source=/dev/null
+. "$ROOT/scripts/lib/common.sh"
+load_cluster_env
+journal_init
+
+{
+  echo "# ${CLUSTER_NAME} — setup journal summary"
+  echo
+  echo "Read raw log on FE: \`${JOURNAL_TSV}\`"
+  echo
+  echo "Last rebuilt: $(date -u -Iseconds 2>/dev/null || date -u)"
+  echo
+  echo "| UTC time | Host | Phase | Action | Status | Detail |"
+  echo "|----------|------|-------|--------|--------|--------|"
+  tail -n +2 "$JOURNAL_TSV" 2>/dev/null | while IFS='	' read -r ts host actor phase action status detail; do
+    printf '| %s | %s | %s | %s | **%s** | %s |\n' \
+      "$ts" "$host" "$phase" "$action" "$status" "$detail"
+  done
+  echo
+  echo "## Latest status per action/host"
+  echo
+  tail -n +2 "$JOURNAL_TSV" 2>/dev/null | awk -F'	' '
+    { key=$2 SUBSEP $5 SUBSEP $4; line=$0; status=$6; if (status != "START") last[key]=line }
+    END { for (k in last) print last[k] }' | sort -t'	' -k1,1 | while IFS='	' read -r ts host actor phase action status detail; do
+    printf -- '- **%s** `%s` / `%s`' "$status" "$host" "$action"
+    [ -n "$detail" ] && printf ' — %s' "$detail"
+    printf '\n'
+  done
+} > "$JOURNAL_SUMMARY"
+
+echo "summary → $JOURNAL_SUMMARY"
--- a/orchestration/sim/cluster0/scripts/lib/common.sh
+++ b/orchestration/sim/cluster0/scripts/lib/common.sh
@@ -0,0 +1,192 @@
+# shellcheck shell=sh
+# Shared helpers for cast cluster scripts.
+
+log() {
+  printf '[%s] %s\n' "$(date -Iseconds 2>/dev/null || date)" "$*"
+}
+
+die() {
+  log "ERROR: $*"
+  if [ -n "${JOURNAL_LAST_ACTION:-}" ]; then
+    journal_record NOK "$JOURNAL_LAST_ACTION" "$*"
+  fi
+  exit 1
+}
+
+cluster_root() {
+  if [ -n "${CAST_CLUSTER_ROOT:-}" ]; then
+    printf '%s\n' "$CAST_CLUSTER_ROOT"
+    return
+  fi
+  if [ -f /shared/cluster/cluster.env ]; then
+    printf '/shared/cluster\n'
+    return
+  fi
+  if [ -f "$(dirname "$0")/../cluster.env" ]; then
+    cd "$(dirname "$0")/.." && pwd
+    return
+  fi
+  die "cannot locate cluster root (is /shared mounted?)"
+}
+
+load_cluster_env() {
+  ROOT="$(cluster_root)"
+  ENV_FILE="${1:-$ROOT/cluster.env}"
+  [ -f "$ENV_FILE" ] || die "missing $ENV_FILE"
+  # shellcheck disable=SC1090
+  . "$ENV_FILE"
+  CAST_CLUSTER_ROOT="$ROOT"
+  FE_NFS="${FE_NFS_SERVER}:${FE_NFS_EXPORT_PATH}"
+  FSTAB_NFS_LINE="${FE_NFS} ${SHARED_MOUNT} nfs rw,_netdev,nofail,noatime,nodiratime 0 0"
+  ALL_CAST_HOSTS="${CAST01_HOST} ${CAST02_HOST} ${CAST03_HOST}"
+  JOURNAL_DIR="${SHARED_MOUNT}/journal"
+  JOURNAL_TSV="${JOURNAL_DIR}/${CLUSTER_NAME}.journal.tsv"
+  JOURNAL_SUMMARY="${JOURNAL_DIR}/${CLUSTER_NAME}.summary.md"
+}
+
+journal_init() {
+  load_cluster_env 2>/dev/null || return 0
+  mkdir -p "$JOURNAL_DIR"
+  if [ ! -f "$JOURNAL_TSV" ]; then
+    printf 'timestamp_utc\thost\tactor\tphase\taction\tstatus\tdetail\n' > "$JOURNAL_TSV"
+  fi
+}
+
+journal_record() {
+  _status="$1"
+  _action="$2"
+  _detail="${3:-}"
+  _phase="${JOURNAL_PHASE:-}"
+  journal_init
+  _ts="$(date -u -Iseconds 2>/dev/null || date -u +%Y-%m-%dT%H:%M:%SZ)"
+  _host="$(host_short 2>/dev/null || echo unknown)"
+  _actor="${JOURNAL_ACTOR:-script}"
+  _line="${_ts}	${_host}	${_actor}	${_phase}	${_action}	${_status}	${_detail}"
+  printf '%s\n' "$_line" >> "$JOURNAL_TSV"
+  log "journal ${_status} ${_action}${_detail:+ — ${_detail}}"
+  if [ -x "${CAST_CLUSTER_ROOT}/scripts/journal-rebuild-summary.sh" ] 2>/dev/null; then
+    sh "${CAST_CLUSTER_ROOT}/scripts/journal-rebuild-summary.sh" 2>/dev/null || true
+  elif [ -f "${CAST_CLUSTER_ROOT}/scripts/journal-rebuild-summary.sh" ]; then
+    sh "${CAST_CLUSTER_ROOT}/scripts/journal-rebuild-summary.sh" 2>/dev/null || true
+  fi
+}
+
+journal_start() {
+  JOURNAL_LAST_ACTION="$1"
+  journal_record START "$1" "${2:-}"
+}
+
+journal_ok() {
+  journal_record OK "$1" "${2:-}"
+  JOURNAL_LAST_ACTION=""
+}
+
+journal_nok() {
+  journal_record NOK "$1" "${2:-}"
+  JOURNAL_LAST_ACTION=""
+}
+
+journal_warn() {
+  journal_record WARN "$1" "${2:-}"
+}
+
+ensure_shared_mounted() {
+  load_cluster_env
+  if mount | grep -q " on ${SHARED_MOUNT} "; then
+    journal_init
+    return 0
+  fi
+  if [ -f /etc/cast-cluster.env ]; then
+    # shellcheck disable=SC1091
+    . /etc/cast-cluster.env
+    FE_NFS="${FE_NFS_SERVER}:${FE_NFS_EXPORT_PATH}"
+    FSTAB_NFS_LINE="${FE_NFS} ${SHARED_MOUNT} nfs rw,_netdev,nofail,noatime,nodiratime 0 0"
+  fi
+  log "mounting ${SHARED_MOUNT} from ${FE_NFS}"
+  journal_start nfs_mount "from ${FE_NFS}"
+  mkdir -p "$SHARED_MOUNT"
+  grep -v "${FE_NFS_EXPORT_PATH}" /etc/fstab > /var/tmp/cast-fstab.new 2>/dev/null || cp /etc/fstab /var/tmp/cast-fstab.new
+  echo "$FSTAB_NFS_LINE" >> /var/tmp/cast-fstab.new
+  mv /var/tmp/cast-fstab.new /etc/fstab
+  rc-update add nfsmount boot 2>/dev/null || true
+  mount "$SHARED_MOUNT" 2>/dev/null || mount -a
+  if mount | grep -q " on ${SHARED_MOUNT} "; then
+    load_cluster_env
+    journal_ok nfs_mount "${SHARED_MOUNT}"
+  else
+    journal_nok nfs_mount "mount failed"
+    die "NFS mount failed: ${SHARED_MOUNT}"
+  fi
+}
+
+run_script() {
+  _script="$1"
+  _action="$(basename "$_script" .sh)"
+  journal_start "$_action" "$_script"
+  if sh "$_script"; then
+    journal_ok "$_action" "$_script"
+    return 0
+  fi
+  journal_nok "$_action" "$_script"
+  return 1
+}
+
+host_short() {
+  hostname -s 2>/dev/null || hostname
+}
+
+is_primary_node() {
+  [ "$(host_short)" = "$CAST01_HOST" ]
+}
+
+is_replica_node() {
+  H="$(host_short)"
+  [ "$H" = "$CAST02_HOST" ] || [ "$H" = "$CAST03_HOST" ]
+}
+
+wait_for_tcp() {
+  _host="$1"
+  _port="$2"
+  _timeout="${3:-300}"
+  _i=0
+  while [ "$_i" -lt "$_timeout" ]; do
+    if nc -z "$_host" "$_port" 2>/dev/null; then
+      return 0
+    fi
+    _i=$((_i + 1))
+    sleep 1
+  done
+  die "timeout waiting for ${_host}:${_port}"
+}
+
+wait_for_primary_db() {
+  load_cluster_env
+  CREDS="${CAST_CLUSTER_ROOT}/credentials.txt"
+  APP_PASS="$(awk '/^\[mariadb_app\]/{f=1;next} /^\[/{f=0} f&&/^password=/{print substr($0,10); exit}' "$CREDS")"
+  _i=0
+  while [ "$_i" -lt 300 ]; do
+    if mariadb -u androidcast -p"$APP_PASS" -h"$PRIMARY_DB_HOST" -N -e 'SELECT 1' >/dev/null 2>&1; then
+      return 0
+    fi
+    _i=$((_i + 1))
+    sleep 2
+  done
+  die "primary DB not reachable at ${PRIMARY_DB_HOST}"
+}
+
+read_cred() {
+  _section="$1"
+  _key="$2"
+  _file="${CAST_CLUSTER_ROOT}/credentials.txt"
+  awk -v s="[$_section]" -v k="$_key=" '
+    $0 == s { f=1; next }
+    /^\[/ { f=0 }
+    f && index($0, k) == 1 { print substr($0, length(k)+1); exit }
+  ' "$_file"
+}
+
+ssh_node() {
+  _host="$1"
+  shift
+  ssh -o BatchMode=yes -o StrictHostKeyChecking=no "${SSH_USER}@${_host}" "$@"
+}
--- a/orchestration/sim/cluster0/scripts/load-schemas.sh
+++ b/orchestration/sim/cluster0/scripts/load-schemas.sh
@@ -0,0 +1,39 @@
+#!/bin/sh
+# Load project schemas on cast01 primary only.
+set -eu
+SHARED=/shared/cluster/sql
+[ "$(hostname -s)" = cast01 ] || { echo "run on cast01 only"; exit 1; }
+
+apply() {
+  f="$1"
+  [ -f "$f" ] || { echo "missing $f"; exit 1; }
+  echo "==> $f"
+  mariadb -u root < "$f"
+}
+
+apply_db() {
+  db="$1"
+  f="$2"
+  echo "==> $db < $f"
+  mariadb -u root "$db" < "$f"
+}
+
+apply "$SHARED/crashes/schema.mariadb.sql"
+
+# Migrations after base schema (003 is for legacy; base schema already includes tickets)
+for m in \
+  004_ticket_workflow.sql \
+  005_ticket_attachments.sql \
+  006_graph_sessions.sql \
+  007_remote_access.sql \
+  008_graph_sessions_indexes.sql \
+  008_auth_email_2fa.sql \
+  009_rssh_sessions.sql
+do
+  apply_db androidcast_crashes "$SHARED/crashes/migrations/$m"
+done
+
+apply "$SHARED/url_shortener/schema.mariadb.sql"
+
+mariadb -u root -e "SHOW DATABASES; SELECT COUNT(*) AS crash_tables FROM information_schema.tables WHERE table_schema='androidcast_crashes'; SELECT COUNT(*) AS url_tables FROM information_schema.tables WHERE table_schema='url_shortener';"
+echo "schemas_loaded_ok"
--- a/orchestration/sim/cluster0/scripts/mariadb-primary.sh
+++ b/orchestration/sim/cluster0/scripts/mariadb-primary.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+# Configure cast01 as MariaDB GTID primary. Idempotent-ish.
+set -eu
+SHARED=/shared/cluster
+CREDS="$SHARED/credentials.txt"
+REPL_USER="$(awk '/^\[mariadb_replication\]/{f=1;next} /^\[/{f=0} f&&/^user=/{print substr($0,6); exit}' "$CREDS")"
+REPL_PASS="$(awk '/^\[mariadb_replication\]/{f=1;next} /^\[/{f=0} f&&/^password=/{print substr($0,10); exit}' "$CREDS")"
+APP_USER="$(awk '/^\[mariadb_app\]/{f=1;next} /^\[/{f=0} f&&/^user=/{print substr($0,6); exit}' "$CREDS")"
+APP_PASS="$(awk '/^\[mariadb_app\]/{f=1;next} /^\[/{f=0} f&&/^password=/{print substr($0,10); exit}' "$CREDS")"
+
+[ "$(hostname -s)" = cast01 ] || { echo "run on cast01 only"; exit 1; }
+
+grep -q '^skip-networking' /etc/my.cnf.d/mariadb-server.cnf && \
+  sed -i 's/^skip-networking/#skip-networking/' /etc/my.cnf.d/mariadb-server.cnf
+
+install -D -m 644 "$SHARED/mariadb/primary.cnf" /etc/my.cnf.d/lab-cluster-primary.cnf
+rc-update add mariadb default 2>/dev/null || true
+rc-service mariadb restart
+
+mariadb -u root <<SQL
+CREATE USER IF NOT EXISTS '${REPL_USER}'@'10.7.16.%' IDENTIFIED BY '${REPL_PASS}';
+GRANT REPLICATION SLAVE, REPLICATION CLIENT, RELOAD, LOCK TABLES, SELECT, SHOW VIEW, EVENT, TRIGGER ON *.* TO '${REPL_USER}'@'10.7.16.%';
+CREATE USER IF NOT EXISTS '${APP_USER}'@'10.7.16.%' IDENTIFIED BY '${APP_PASS}';
+CREATE USER IF NOT EXISTS '${APP_USER}'@'localhost' IDENTIFIED BY '${APP_PASS}';
+GRANT ALL PRIVILEGES ON androidcast_crashes.* TO '${APP_USER}'@'10.7.16.%';
+GRANT ALL PRIVILEGES ON androidcast_crashes.* TO '${APP_USER}'@'localhost';
+GRANT ALL PRIVILEGES ON url_shortener.* TO '${APP_USER}'@'10.7.16.%';
+GRANT ALL PRIVILEGES ON url_shortener.* TO '${APP_USER}'@'localhost';
+FLUSH PRIVILEGES;
+SQL
+
+echo "primary_ok server_id=$(mariadb -u root -N -e 'SELECT @@server_id')"
+echo "gtid=$(mariadb -u root -N -e 'SELECT @@gtid_current_pos')"
--- a/orchestration/sim/cluster0/scripts/mariadb-replica.sh
+++ b/orchestration/sim/cluster0/scripts/mariadb-replica.sh
@@ -0,0 +1,73 @@
+#!/bin/sh
+# Bootstrap cast02 or cast03 as GTID replica of cast01.
+set -eu
+set -o pipefail
+SHARED=/shared/cluster
+CREDS="$SHARED/credentials.txt"
+[ -f "$SHARED/cluster.env" ] && . "$SHARED/cluster.env"
+HOST="$(hostname -s)"
+case "$HOST" in
+  cast02) SERVER_ID=2 ;;
+  cast03) SERVER_ID=3 ;;
+  *) echo "run on cast02 or cast03"; exit 1 ;;
+esac
+
+REPL_PASS="$(awk '/^\[mariadb_replication\]/{f=1;next} /^\[/{f=0} f&&/^password=/{print substr($0,10); exit}' "$CREDS")"
+REPL_USER="$(awk '/^\[mariadb_replication\]/{f=1;next} /^\[/{f=0} f&&/^user=/{print substr($0,6); exit}' "$CREDS")"
+APP_PASS="$(awk '/^\[mariadb_app\]/{f=1;next} /^\[/{f=0} f&&/^password=/{print substr($0,10); exit}' "$CREDS")"
+APP_USER="$(awk '/^\[mariadb_app\]/{f=1;next} /^\[/{f=0} f&&/^user=/{print substr($0,6); exit}' "$CREDS")"
+
+grep -q '^skip-networking' /etc/my.cnf.d/mariadb-server.cnf && \
+  sed -i 's/^skip-networking/#skip-networking/' /etc/my.cnf.d/mariadb-server.cnf
+
+apk add --no-cache mariadb
+install -D -m 644 "$SHARED/mariadb/replica.cnf" /etc/my.cnf.d/lab-cluster-replica.cnf
+grep -q '^server-id=' /etc/my.cnf.d/lab-cluster-replica.cnf && \
+  sed -i "s/^server-id=.*/server-id=${SERVER_ID}/" /etc/my.cnf.d/lab-cluster-replica.cnf || \
+  echo "server-id=${SERVER_ID}" >> /etc/my.cnf.d/lab-cluster-replica.cnf
+
+rc-service mariadb stop 2>/dev/null || true
+rm -rf /var/lib/mysql/*
+/etc/init.d/mariadb setup
+
+rc-update add mariadb default
+rc-service mariadb restart
+sleep 2
+
+echo "=== app DB sync from cast01 (skip mysql.* — avoids help_topic dump failures) ==="
+mariadb-dump --skip-ssl -h cast01 -u "${REPL_USER}" -p"${REPL_PASS}" \
+  --databases androidcast_crashes url_shortener \
+  --single-transaction --gtid --routines --events \
+  | mariadb -u root
+
+GTID_POS="$(mariadb --skip-ssl -h cast01 -u "${REPL_USER}" -p"${REPL_PASS}" -N -e 'SELECT @@gtid_current_pos')"
+[ -n "$GTID_POS" ] || { echo "FAIL: could not read gtid from cast01"; exit 1; }
+mariadb -u root -e "SET GLOBAL gtid_slave_pos='${GTID_POS}'"
+echo "gtid_slave_pos set to: ${GTID_POS}"
+
+mariadb -u root <<SQL
+CREATE USER IF NOT EXISTS '${APP_USER}'@'10.7.16.%' IDENTIFIED BY '${APP_PASS}';
+CREATE USER IF NOT EXISTS '${APP_USER}'@'localhost' IDENTIFIED BY '${APP_PASS}';
+GRANT ALL PRIVILEGES ON androidcast_crashes.* TO '${APP_USER}'@'10.7.16.%';
+GRANT ALL PRIVILEGES ON androidcast_crashes.* TO '${APP_USER}'@'localhost';
+GRANT ALL PRIVILEGES ON url_shortener.* TO '${APP_USER}'@'10.7.16.%';
+GRANT ALL PRIVILEGES ON url_shortener.* TO '${APP_USER}'@'localhost';
+FLUSH PRIVILEGES;
+CHANGE MASTER TO
+  MASTER_HOST='cast01',
+  MASTER_USER='${REPL_USER}',
+  MASTER_PASSWORD='${REPL_PASS}',
+  MASTER_USE_GTID=slave_pos;
+START REPLICA;
+SQL
+
+sleep 3
+mariadb -u root -e "SHOW REPLICA STATUS\G" | grep -E 'Slave_IO_Running|Slave_SQL_Running|Last_Error|Last_SQL_Error|Seconds_Behind_Master'
+mariadb -u root -e "SELECT table_schema, COUNT(*) AS tables FROM information_schema.tables WHERE table_schema IN ('androidcast_crashes','url_shortener') GROUP BY table_schema;"
+URL_N="$(mariadb -u root -N -e 'SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='"'"'url_shortener'"'"'')"
+CRASH_N="$(mariadb -u root -N -e 'SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='"'"'androidcast_crashes'"'"'')"
+EXP_CRASH="${EXPECTED_CRASHES_TABLES:-16}"
+EXP_URL="${EXPECTED_URL_SHORTENER_TABLES:-4}"
+[ "$CRASH_N" = "$EXP_CRASH" ] && [ "$URL_N" = "$EXP_URL" ] || { echo "FAIL: expected ${EXP_CRASH}+${EXP_URL} tables, got crashes=$CRASH_N url=$URL_N"; exit 1; }
+mariadb -u root -e "SHOW REPLICA STATUS\G" | grep -E 'Slave_SQL_Running: Yes' >/dev/null || { echo "FAIL: SQL thread not running"; mariadb -u root -e "SHOW REPLICA STATUS\G" | grep -E 'Last_SQL_Error'; exit 1; }
+echo "replica_ok $HOST server_id=$SERVER_ID"
--- a/orchestration/sim/cluster0/scripts/verify-cluster.sh
+++ b/orchestration/sim/cluster0/scripts/verify-cluster.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+# Quick health check for cast cluster.
+set -eu
+HOST="$(hostname -s)"
+FAIL=0
+echo "=== $HOST ==="
+mount | grep ' /shared ' || { echo "WARN: /shared not mounted"; FAIL=1; }
+curl -sS -o /dev/null -w "nginx=%{http_code}\n" http://127.0.0.1/ || true
+php83 -v | head -1
+
+case "$HOST" in
+  cast01)
+    mariadb -u root -e "SELECT @@server_id AS server_id, @@read_only AS read_only, @@gtid_current_pos AS gtid;"
+    mariadb -u androidcast -pcast-cluster-dev -h127.0.0.1 -e \
+      "SELECT table_schema, COUNT(*) AS tables FROM information_schema.tables WHERE table_schema IN ('androidcast_crashes','url_shortener') GROUP BY table_schema;"
+    ;;
+  cast02|cast03)
+    mariadb -u root -e "SELECT @@server_id AS server_id, @@read_only AS read_only;" 2>/dev/null || { echo "mariadb not running"; FAIL=1; }
+    RS="$(mariadb -u root -e 'SHOW REPLICA STATUS\G' 2>/dev/null || true)"
+    echo "$RS" | grep -E 'Master_Host|Slave_IO_Running|Slave_SQL_Running|Seconds_Behind_Master|Last_SQL_Error' || true
+    echo "$RS" | grep -E 'Slave_IO_Running: Yes' >/dev/null || { echo "FAIL: IO thread not running"; FAIL=1; }
+    echo "$RS" | grep -E 'Slave_SQL_Running: Yes' >/dev/null || { echo "FAIL: SQL thread not running"; FAIL=1; }
+    mariadb -u root -e \
+      "SELECT table_schema, COUNT(*) AS tables FROM information_schema.tables WHERE table_schema IN ('androidcast_crashes','url_shortener') GROUP BY table_schema;"
+    mariadb -u androidcast -pcast-cluster-dev -h127.0.0.1 -e "SELECT 1 AS local_read_ok;" 2>/dev/null || echo "WARN: local app user read failed"
+    ;;
+esac
+
+if [ "$FAIL" -eq 0 ]; then
+  echo OK
+else
+  echo FAIL
+  exit 1
+fi
--- a/orchestration/sim/cluster0/verify_global_setup.sh
+++ b/orchestration/sim/cluster0/verify_global_setup.sh
@@ -0,0 +1,64 @@
+#!/bin/sh
+# Post-DNS / post-FE verification — run after PUBLIC_ORIGIN points at this cluster.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")" && pwd)"
+export CAST_CLUSTER_ROOT="$ROOT"
+# shellcheck source=/dev/null
+. "$ROOT/scripts/lib/common.sh"
+export JOURNAL_ACTOR=verify_global
+export JOURNAL_PHASE=verify_global
+ensure_shared_mounted
+
+FAIL=0
+ORIGIN="${PUBLIC_ORIGIN%/}"
+
+check_url() {
+  _path="$1"
+  _expect="${2:-200}"
+  _url="${ORIGIN}${_path}"
+  _action="curl ${_path}"
+  journal_start "$_action" "$_url"
+  _code="$(curl -sS -o /dev/null -w '%{http_code}' -L --max-time 30 "$_url" 2>/dev/null || echo 000)"
+  if [ "$_code" = "$_expect" ]; then
+    journal_ok "$_action" "http=${_code}"
+    log "OK $_code $_url"
+  else
+    journal_nok "$_action" "got=${_code} want=${_expect}"
+    log "FAIL got $_code want $_expect $_url"
+    FAIL=1
+  fi
+}
+
+journal_start verify_global "origin=${ORIGIN}"
+log "verify_global_setup cluster=${CLUSTER_NAME} origin=${ORIGIN}"
+[ -n "$ORIGIN" ] || die "set PUBLIC_ORIGIN in cluster.env"
+
+check_url "${APP_HUB_PATH}/" "200"
+check_url "${APP_BASE_PATH}/" "200"
+check_url "${APP_BASE_PATH}/assets/js/app.js" "200"
+check_url "${APP_BASE_PATH}/api/diag.php" "200"
+
+# Short links UI (may redirect to login)
+_code="$(curl -sS -o /dev/null -w '%{http_code}' -L --max-time 30 "${ORIGIN}${APP_BASE_PATH}/?view=short_links" 2>/dev/null || echo 000)"
+case "$_code" in
+  200|302) log "OK $_code short_links" ;;
+  *) log "FAIL short_links http=$_code"; FAIL=1 ;;
+esac
+
+if [ -n "${FE_PROXY_TARGET:-}" ]; then
+  _fe_code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 15 "http://${FE_PROXY_TARGET}${APP_BASE_PATH}/" 2>/dev/null || echo 000)"
+  case "$_fe_code" in
+    200|302) log "OK upstream $_fe_code http://${FE_PROXY_TARGET}${APP_BASE_PATH}/" ;;
+    *) log "WARN upstream $_fe_code (check FE nginx after DNS cutover)" ;;
+  esac
+fi
+
+if [ "$FAIL" -eq 0 ]; then
+  journal_ok verify_global
+  log "verify_global_setup_ok ${CLUSTER_NAME}"
+  exit 0
+fi
+journal_nok verify_global
+log "verify_global_setup_FAIL ${CLUSTER_NAME}"
+exit 1
--- a/orchestration/sim/cluster0/verify_lab_setup.sh
+++ b/orchestration/sim/cluster0/verify_lab_setup.sh
@@ -0,0 +1,105 @@
+#!/bin/sh
+# Internal cluster verification (before or after DNS). No external network required for --local.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")" && pwd)"
+export CAST_CLUSTER_ROOT="$ROOT"
+# shellcheck source=/dev/null
+. "$ROOT/scripts/lib/common.sh"
+export JOURNAL_ACTOR=verify_lab
+export JOURNAL_PHASE=verify
+
+MODE=local
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --local|--local-only) MODE=local; shift ;;
+    --cluster) MODE=cluster; shift ;;
+    *) die "unknown arg: $1" ;;
+  esac
+done
+
+FAIL=0
+check_local_node() {
+  H="$(host_short)"
+  log "verify local $H"
+  run_script "$ROOT/scripts/verify-cluster.sh" || FAIL=1
+
+  nginx -t >/dev/null 2>&1 || { log "FAIL nginx -t"; FAIL=1; }
+  rc-status nginx php-fpm83 2>/dev/null | grep -E 'nginx|php-fpm83' || true
+  if is_primary_node || is_replica_node; then
+    rc-service mariadb status 2>/dev/null | grep -q started || { log "FAIL mariadb not running"; FAIL=1; }
+  fi
+
+  CODE="$(curl -sS -o /dev/null -w '%{http_code}' "http://127.0.0.1${APP_BASE_PATH}/" 2>/dev/null || echo 000)"
+  case "$CODE" in
+    200|302) log "app_http=${CODE}" ;;
+    *) log "FAIL app_http=${CODE} for ${APP_BASE_PATH}/"; FAIL=1 ;;
+  esac
+
+  ASSET="${APP_BASE_PATH}/assets/js/app.js"
+  ACODE="$(curl -sS -o /dev/null -w '%{http_code}' "http://127.0.0.1${ASSET}" 2>/dev/null || echo 000)"
+  case "$ACODE" in
+    200) log "asset_http=${ACODE}" ;;
+    *) log "WARN asset_http=${ACODE} (${ASSET})" ;;
+  esac
+
+  DIAG="$(curl -sS "http://127.0.0.1${APP_BASE_PATH}/api/diag.php" 2>/dev/null || true)"
+  echo "$DIAG" | grep -q '"ok":true' || { log "FAIL diag.php"; FAIL=1; }
+
+  CRASH_N="$(mariadb -u androidcast -p"$(read_cred mariadb_app password)" -h127.0.0.1 -N -e \
+    "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='androidcast_crashes'" 2>/dev/null || echo 0)"
+  [ "$CRASH_N" = "$EXPECTED_CRASHES_TABLES" ] || { log "FAIL crashes tables=$CRASH_N want $EXPECTED_CRASHES_TABLES"; FAIL=1; }
+}
+
+check_remote_node_http() {
+  _host="$1"
+  _ip=""
+  case "$_host" in
+    "$CAST01_HOST") _ip="$CAST01_IP" ;;
+    "$CAST02_HOST") _ip="$CAST02_IP" ;;
+    "$CAST03_HOST") _ip="$CAST03_IP" ;;
+  esac
+  [ -n "$_ip" ] || { log "FAIL unknown host $_host"; FAIL=1; return; }
+  log "verify http $_host ($_ip)"
+  _code="$(curl -sS -o /dev/null -w '%{http_code}' "http://${_ip}${APP_BASE_PATH}/" 2>/dev/null || echo 000)"
+  case "$_code" in
+    200|302) log "OK $_host app_http=$_code" ;;
+    *) log "FAIL $_host app_http=$_code"; FAIL=1 ;;
+  esac
+  _diag="$(curl -sS "http://${_ip}${APP_BASE_PATH}/api/diag.php" 2>/dev/null || true)"
+  echo "$_diag" | grep -q '"ok":true' || { log "FAIL $_host diag.php"; FAIL=1; }
+}
+
+check_remote_node_ssh() {
+  _host="$1"
+  log "verify ssh $_host"
+  ssh_node "$_host" "sudo sh ${SHARED_MOUNT}/cluster/verify_lab_setup.sh --local" || FAIL=1
+}
+
+ensure_shared_mounted
+journal_start verify_lab "mode=${MODE}"
+
+if [ "$MODE" = cluster ]; then
+  for H in $CAST01_HOST $CAST02_HOST $CAST03_HOST; do
+    if [ "$(host_short)" = "$H" ]; then
+      check_local_node
+    else
+      if ssh -o BatchMode=yes -o ConnectTimeout=3 "${SSH_USER}@${H}" true 2>/dev/null; then
+        check_remote_node_ssh "$H"
+      else
+        check_remote_node_http "$H"
+      fi
+    fi
+  done
+else
+  check_local_node
+fi
+
+if [ "$FAIL" -eq 0 ]; then
+  journal_ok verify_lab "mode=${MODE}"
+  log "verify_lab_setup_ok ${CLUSTER_NAME} mode=$MODE"
+  exit 0
+fi
+journal_nok verify_lab "mode=${MODE}"
+log "verify_lab_setup_FAIL ${CLUSTER_NAME} mode=$MODE"
+exit 1