initial

2026-07-29 03:58:34 +03:00 · 2026-06-23 12:21:14 +02:00
commit 7f3e4fea9f
53 changed files with 2725 additions and 0 deletions
--- a/sim/cluster0/scripts/baseline.sh
+++ b/sim/cluster0/scripts/baseline.sh
@@ -0,0 +1,67 @@
+#!/bin/sh
+# cast cluster baseline — packages, NFS, hosts, nginx/php smoke vhost.
+set -eu
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+# shellcheck source=/dev/null
+. "$ROOT/scripts/lib/common.sh"
+ensure_shared_mounted
+
+HOST="$(host_short)"
+STAGE="$(mktemp -d /var/tmp/cast-cluster.XXXXXX)"
+trap 'rm -rf "$STAGE"' EXIT
+
+log "baseline start on $HOST"
+
+grep -v "${FE_NFS_EXPORT_PATH}" /etc/fstab > "$STAGE/fstab.new" || true
+echo "$FSTAB_NFS_LINE" >> "$STAGE/fstab.new"
+mv "$STAGE/fstab.new" /etc/fstab
+mkdir -p "$SHARED_MOUNT"
+rc-update add nfsmount boot 2>/dev/null || true
+mount "$SHARED_MOUNT" 2>/dev/null || mount -a 2>/dev/null || true
+
+grep -v '# cast-cluster' /etc/hosts | grep -v 'cast0[123]\.' | grep -v 'artc0\.' > "$STAGE/hosts.new" || cp /etc/hosts "$STAGE/hosts.new"
+cat >> "$STAGE/hosts.new" <<EOF
+# cast-cluster ${CLUSTER_NAME}
+${CAST01_IP} ${CAST01_HOST} ${CAST01_HOST}.${CAST_DOMAIN}
+${CAST02_IP} ${CAST02_HOST} ${CAST02_HOST}.${CAST_DOMAIN}
+${CAST03_IP} ${CAST03_HOST} ${CAST03_HOST}.${CAST_DOMAIN}
+${ARTC0_IP} ${ARTC0_HOST} ${ARTC0_HOST}.${CAST_DOMAIN} alpine-be
+EOF
+mv "$STAGE/hosts.new" /etc/hosts
+
+apk update
+PKGS="$APK_PACKAGES_BASE"
+if is_primary_node; then
+  PKGS="$PKGS mariadb"
+fi
+# shellcheck disable=SC2086
+apk add --no-cache $PKGS
+
+rc-update add nginx default 2>/dev/null || true
+rc-update add php-fpm83 default 2>/dev/null || true
+run_script "$ROOT/scripts/configure-php-fpm.sh"
+rc-service php-fpm83 restart 2>/dev/null || rc-service php-fpm83 start 2>/dev/null || true
+rc-service nginx start 2>/dev/null || true
+
+if is_primary_node; then
+  grep -q '^skip-networking' /etc/my.cnf.d/mariadb-server.cnf 2>/dev/null && \
+    sed -i 's/^skip-networking/#skip-networking/' /etc/my.cnf.d/mariadb-server.cnf || true
+  grep -q '^bind-address' /etc/my.cnf.d/mariadb-server.cnf 2>/dev/null || \
+    printf '\n[mysqld]\nbind-address = 0.0.0.0\n' >> /etc/my.cnf.d/mariadb-server.cnf
+  rc-update add mariadb default 2>/dev/null || true
+  rc-service mariadb restart 2>/dev/null || rc-service mariadb start 2>/dev/null || true
+fi
+
+mkdir -p /var/www/localhost/htdocs
+echo "<html><body><h1>${HOST}</h1><p>${CLUSTER_NAME} node</p></body></html>" > /var/www/localhost/htdocs/index.html
+chown -R nginx:nginx /var/www/localhost/htdocs 2>/dev/null || true
+
+mkdir -p "$ROOT"
+IP="$(ip -4 -o addr show eth0 2>/dev/null | awk '{print $4}' | cut -d/ -f1)"
+echo "$HOST $(date -Iseconds) ip=$IP cluster=$CLUSTER_NAME" >> "$ROOT/seen.log"
+printf '{"host":"%s","ip":"%s","cluster":"%s","configured_at":"%s"}\n' \
+  "$HOST" "$IP" "$CLUSTER_NAME" "$(date -Iseconds)" > "$ROOT/${HOST}.json"
+
+mount | grep -q " on ${SHARED_MOUNT} " || die "NFS not mounted"
+curl -sS -o /dev/null -w "nginx_http=%{http_code}\n" http://127.0.0.1/
+log "baseline_ok $HOST"
--- a/sim/cluster0/scripts/configure-nginx.sh
+++ b/sim/cluster0/scripts/configure-nginx.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+set -eu
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+# shellcheck source=/dev/null
+. "$ROOT/scripts/lib/common.sh"
+ensure_shared_mounted
+
+install -D -m 644 "$ROOT/nginx/apps-port80.conf" /etc/nginx/http.d/androidcast.conf
+rm -f /etc/nginx/http.d/default.conf /etc/nginx/http.d/cluster.conf 2>/dev/null || true
+nginx -t
+rc-service nginx reload
+log "nginx androidcast vhost installed"
--- a/sim/cluster0/scripts/configure-php-fpm.sh
+++ b/sim/cluster0/scripts/configure-php-fpm.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+set -eu
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+# shellcheck source=/dev/null
+. "$ROOT/scripts/lib/common.sh"
+ensure_shared_mounted
+
+CONF=/etc/php83/php-fpm.d/www.conf
+grep -q '^listen = 127.0.0.1:9000' "$CONF" 2>/dev/null || \
+  sed -i 's|^listen = .*|listen = 127.0.0.1:9000|' "$CONF"
+grep -q '^listen.allowed_clients' "$CONF" 2>/dev/null || \
+  printf '\nlisten.allowed_clients = 127.0.0.1\n' >> "$CONF"
+log "php-fpm83 listen 127.0.0.1:9000"
--- a/sim/cluster0/scripts/deploy-app.sh
+++ b/sim/cluster0/scripts/deploy-app.sh
@@ -0,0 +1,109 @@
+#!/bin/sh
+# Deploy androidcast project tree + lab config.php (local disk, not NFS).
+set -eu
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+# shellcheck source=/dev/null
+. "$ROOT/scripts/lib/common.sh"
+ensure_shared_mounted
+
+APP_PASS="$(read_cred mariadb_app password)"
+DB_HOST="$PRIMARY_DB_HOST"
+if is_primary_node; then
+  DB_HOST=127.0.0.1
+fi
+
+mkdir -p "$(dirname "$APP_ROOT")"
+GIT_ROOT="${APP_ROOT}/android_cast"
+BACKEND="${GIT_ROOT}/examples/crash_reporter/backend"
+CONFIG="${BACKEND}/config/config.php"
+
+case "$APP_SOURCE" in
+  git)
+    if [ -d "${GIT_ROOT}/.git" ]; then
+      log "git pull ${GIT_BRANCH} in ${GIT_ROOT}"
+      git config --global --add safe.directory "$GIT_ROOT" 2>/dev/null || true
+      git -C "$GIT_ROOT" fetch origin
+      git -C "$GIT_ROOT" checkout "$GIT_BRANCH"
+      git -C "$GIT_ROOT" pull --ff-only origin "$GIT_BRANCH"
+    else
+      log "git clone ${GIT_ORIGIN} → ${GIT_ROOT}"
+      rm -rf "$GIT_ROOT"
+      git clone --branch "$GIT_BRANCH" --depth 1 "$GIT_ORIGIN" "$GIT_ROOT"
+      git config --global --add safe.directory "$GIT_ROOT" 2>/dev/null || true
+    fi
+    ;;
+  bundle)
+    BUNDLE="${ROOT}/release/current"
+    [ -d "$BUNDLE" ] || die "missing bundle dir $BUNDLE (APP_SOURCE=bundle)"
+    log "rsync bundle $BUNDLE → ${APP_ROOT}"
+    mkdir -p "$APP_ROOT"
+    rsync -a --delete "$BUNDLE/" "$APP_ROOT/"
+    ;;
+  *)
+    die "unknown APP_SOURCE=$APP_SOURCE"
+    ;;
+esac
+
+mkdir -p "${BACKEND}/config" "${BACKEND}/data" "${BACKEND}/storage"
+chown -R nginx:nginx "$APP_ROOT" 2>/dev/null || true
+
+if [ ! -f "$CONFIG" ] || [ "${FORCE_CONFIG:-1}" = "1" ]; then
+  log "writing lab config.php"
+  cat > "$CONFIG" <<PHP
+<?php
+return [
+    'app_name' => 'Android Cast Issues',
+    'base_path' => '${APP_BASE_PATH}',
+    'db' => [
+        'driver' => 'mariadb',
+        'sqlite_path' => __DIR__ . '/../data/crashes.sqlite',
+        'mysql' => [
+            'host' => '${DB_HOST}',
+            'port' => ${PRIMARY_DB_PORT},
+            'socket' => '',
+            'database' => 'androidcast_crashes',
+            'username' => 'androidcast',
+            'password' => '${APP_PASS}',
+            'charset' => 'utf8mb4',
+        ],
+    ],
+    'session_name' => 'ac_crash_sess',
+    'session_cookie_path' => '${APP_HUB_PATH}',
+    'debug' => true,
+    'rbac' => [
+        'default_company_slug' => 'default',
+        'default_company_id' => 1,
+    ],
+    'analytics' => ['enabled' => false, 'measurement_id' => '', 'debug' => false],
+    'remote_access' => [
+        'wg_endpoint' => '',
+        'wg_server_public_key' => '',
+        'provision_peers' => false,
+        'require_wg_tools' => false,
+    ],
+    'auth' => [
+        'encryption_key' => 'lab-cluster-dev-32-char-min-secret!!',
+        'max_attempts' => 8,
+        'lockout_window_minutes' => 15,
+    ],
+    'url_shortener' => [
+        'enabled' => true,
+        'public_base' => '${SHORT_LINKS_PUBLIC_BASE}',
+        'db' => [
+            'mysql' => [
+                'host' => '${DB_HOST}',
+                'port' => ${PRIMARY_DB_PORT},
+                'socket' => '',
+                'database' => 'url_shortener',
+                'username' => 'androidcast',
+                'password' => '${APP_PASS}',
+                'charset' => 'utf8mb4',
+            ],
+        ],
+    ],
+];
+PHP
+  chown nginx:nginx "$CONFIG" 2>/dev/null || true
+fi
+
+log "deploy-app_ok $(host_short)"
--- a/sim/cluster0/scripts/journal-backfill-cluster0.sh
+++ b/sim/cluster0/scripts/journal-backfill-cluster0.sh
@@ -0,0 +1,57 @@
+#!/bin/sh
+# One-time backfill of cluster0 journal from known-good state (2026-06-13 session).
+# Safe to re-run: appends only if journal is empty.
+set -eu
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+# shellcheck source=/dev/null
+. "$ROOT/scripts/lib/common.sh"
+ensure_shared_mounted
+journal_init
+
+if [ -f "$JOURNAL_TSV" ] && [ "$(wc -l < "$JOURNAL_TSV")" -gt 1 ]; then
+  log "journal already has entries — skip backfill (see $JOURNAL_TSV)"
+  sh "$ROOT/scripts/journal-rebuild-summary.sh"
+  exit 0
+fi
+
+export JOURNAL_ACTOR=backfill
+TS="2026-06-13T12:00:00+00:00"
+
+append() {
+  _host="$1"
+  _phase="$2"
+  _action="$3"
+  _status="$4"
+  _detail="$5"
+  printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
+    "$TS" "$_host" "$JOURNAL_ACTOR" "$_phase" "$_action" "$_status" "$_detail" >> "$JOURNAL_TSV"
+}
+
+# Infrastructure (all nodes)
+for H in cast01 cast02 cast03; do
+  append "$H" baseline nfs_mount OK "/shared from FE export"
+  append "$H" baseline packages OK "nginx php83-fpm mariadb-client nfs-utils"
+  append "$H" baseline hosts OK "cast-cluster /etc/hosts block"
+done
+
+# MariaDB
+append cast01 db mariadb-primary OK "GTID primary server-id=1"
+append cast01 db load-schemas OK "16 crashes + 4 url_shortener tables"
+append cast02 db mariadb-replica OK "GTID replica IO+SQL running"
+append cast03 db mariadb-replica OK "GTID replica IO+SQL running"
+
+# App + nginx
+for H in cast01 cast02 cast03; do
+  append "$H" app deploy-app OK "git next → android_cast/"
+  append "$H" app configure-nginx OK "apps-port80.conf php83:9000"
+done
+
+# Verification
+append cast01 verify verify_lab OK "mode=local"
+append cast02 verify verify_lab OK "mode=local"
+append cast03 verify verify_lab OK "mode=local"
+append cast01 verify verify_lab OK "mode=cluster HTTP peers"
+append cast01 verify verify_global START "pending DNS/FE cutover"
+
+sh "$ROOT/scripts/journal-rebuild-summary.sh"
+log "backfill_ok → $JOURNAL_TSV"
--- a/sim/cluster0/scripts/journal-rebuild-summary.sh
+++ b/sim/cluster0/scripts/journal-rebuild-summary.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+# Regenerate human-readable OK/NOK summary from journal TSV (safe on FE via NFS read).
+set -eu
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+# shellcheck source=/dev/null
+. "$ROOT/scripts/lib/common.sh"
+load_cluster_env
+journal_init
+
+{
+  echo "# ${CLUSTER_NAME} — setup journal summary"
+  echo
+  echo "Read raw log on FE: \`${JOURNAL_TSV}\`"
+  echo
+  echo "Last rebuilt: $(date -u -Iseconds 2>/dev/null || date -u)"
+  echo
+  echo "| UTC time | Host | Phase | Action | Status | Detail |"
+  echo "|----------|------|-------|--------|--------|--------|"
+  tail -n +2 "$JOURNAL_TSV" 2>/dev/null | while IFS='	' read -r ts host actor phase action status detail; do
+    printf '| %s | %s | %s | %s | **%s** | %s |\n' \
+      "$ts" "$host" "$phase" "$action" "$status" "$detail"
+  done
+  echo
+  echo "## Latest status per action/host"
+  echo
+  tail -n +2 "$JOURNAL_TSV" 2>/dev/null | awk -F'	' '
+    { key=$2 SUBSEP $5 SUBSEP $4; line=$0; status=$6; if (status != "START") last[key]=line }
+    END { for (k in last) print last[k] }' | sort -t'	' -k1,1 | while IFS='	' read -r ts host actor phase action status detail; do
+    printf -- '- **%s** `%s` / `%s`' "$status" "$host" "$action"
+    [ -n "$detail" ] && printf ' — %s' "$detail"
+    printf '\n'
+  done
+} > "$JOURNAL_SUMMARY"
+
+echo "summary → $JOURNAL_SUMMARY"
--- a/sim/cluster0/scripts/lib/common.sh
+++ b/sim/cluster0/scripts/lib/common.sh
@@ -0,0 +1,192 @@
+# shellcheck shell=sh
+# Shared helpers for cast cluster scripts.
+
+log() {
+  printf '[%s] %s\n' "$(date -Iseconds 2>/dev/null || date)" "$*"
+}
+
+die() {
+  log "ERROR: $*"
+  if [ -n "${JOURNAL_LAST_ACTION:-}" ]; then
+    journal_record NOK "$JOURNAL_LAST_ACTION" "$*"
+  fi
+  exit 1
+}
+
+cluster_root() {
+  if [ -n "${CAST_CLUSTER_ROOT:-}" ]; then
+    printf '%s\n' "$CAST_CLUSTER_ROOT"
+    return
+  fi
+  if [ -f /shared/cluster/cluster.env ]; then
+    printf '/shared/cluster\n'
+    return
+  fi
+  if [ -f "$(dirname "$0")/../cluster.env" ]; then
+    cd "$(dirname "$0")/.." && pwd
+    return
+  fi
+  die "cannot locate cluster root (is /shared mounted?)"
+}
+
+load_cluster_env() {
+  ROOT="$(cluster_root)"
+  ENV_FILE="${1:-$ROOT/cluster.env}"
+  [ -f "$ENV_FILE" ] || die "missing $ENV_FILE"
+  # shellcheck disable=SC1090
+  . "$ENV_FILE"
+  CAST_CLUSTER_ROOT="$ROOT"
+  FE_NFS="${FE_NFS_SERVER}:${FE_NFS_EXPORT_PATH}"
+  FSTAB_NFS_LINE="${FE_NFS} ${SHARED_MOUNT} nfs rw,_netdev,nofail,noatime,nodiratime 0 0"
+  ALL_CAST_HOSTS="${CAST01_HOST} ${CAST02_HOST} ${CAST03_HOST}"
+  JOURNAL_DIR="${SHARED_MOUNT}/journal"
+  JOURNAL_TSV="${JOURNAL_DIR}/${CLUSTER_NAME}.journal.tsv"
+  JOURNAL_SUMMARY="${JOURNAL_DIR}/${CLUSTER_NAME}.summary.md"
+}
+
+journal_init() {
+  load_cluster_env 2>/dev/null || return 0
+  mkdir -p "$JOURNAL_DIR"
+  if [ ! -f "$JOURNAL_TSV" ]; then
+    printf 'timestamp_utc\thost\tactor\tphase\taction\tstatus\tdetail\n' > "$JOURNAL_TSV"
+  fi
+}
+
+journal_record() {
+  _status="$1"
+  _action="$2"
+  _detail="${3:-}"
+  _phase="${JOURNAL_PHASE:-}"
+  journal_init
+  _ts="$(date -u -Iseconds 2>/dev/null || date -u +%Y-%m-%dT%H:%M:%SZ)"
+  _host="$(host_short 2>/dev/null || echo unknown)"
+  _actor="${JOURNAL_ACTOR:-script}"
+  _line="${_ts}	${_host}	${_actor}	${_phase}	${_action}	${_status}	${_detail}"
+  printf '%s\n' "$_line" >> "$JOURNAL_TSV"
+  log "journal ${_status} ${_action}${_detail:+ — ${_detail}}"
+  if [ -x "${CAST_CLUSTER_ROOT}/scripts/journal-rebuild-summary.sh" ] 2>/dev/null; then
+    sh "${CAST_CLUSTER_ROOT}/scripts/journal-rebuild-summary.sh" 2>/dev/null || true
+  elif [ -f "${CAST_CLUSTER_ROOT}/scripts/journal-rebuild-summary.sh" ]; then
+    sh "${CAST_CLUSTER_ROOT}/scripts/journal-rebuild-summary.sh" 2>/dev/null || true
+  fi
+}
+
+journal_start() {
+  JOURNAL_LAST_ACTION="$1"
+  journal_record START "$1" "${2:-}"
+}
+
+journal_ok() {
+  journal_record OK "$1" "${2:-}"
+  JOURNAL_LAST_ACTION=""
+}
+
+journal_nok() {
+  journal_record NOK "$1" "${2:-}"
+  JOURNAL_LAST_ACTION=""
+}
+
+journal_warn() {
+  journal_record WARN "$1" "${2:-}"
+}
+
+ensure_shared_mounted() {
+  load_cluster_env
+  if mount | grep -q " on ${SHARED_MOUNT} "; then
+    journal_init
+    return 0
+  fi
+  if [ -f /etc/cast-cluster.env ]; then
+    # shellcheck disable=SC1091
+    . /etc/cast-cluster.env
+    FE_NFS="${FE_NFS_SERVER}:${FE_NFS_EXPORT_PATH}"
+    FSTAB_NFS_LINE="${FE_NFS} ${SHARED_MOUNT} nfs rw,_netdev,nofail,noatime,nodiratime 0 0"
+  fi
+  log "mounting ${SHARED_MOUNT} from ${FE_NFS}"
+  journal_start nfs_mount "from ${FE_NFS}"
+  mkdir -p "$SHARED_MOUNT"
+  grep -v "${FE_NFS_EXPORT_PATH}" /etc/fstab > /var/tmp/cast-fstab.new 2>/dev/null || cp /etc/fstab /var/tmp/cast-fstab.new
+  echo "$FSTAB_NFS_LINE" >> /var/tmp/cast-fstab.new
+  mv /var/tmp/cast-fstab.new /etc/fstab
+  rc-update add nfsmount boot 2>/dev/null || true
+  mount "$SHARED_MOUNT" 2>/dev/null || mount -a
+  if mount | grep -q " on ${SHARED_MOUNT} "; then
+    load_cluster_env
+    journal_ok nfs_mount "${SHARED_MOUNT}"
+  else
+    journal_nok nfs_mount "mount failed"
+    die "NFS mount failed: ${SHARED_MOUNT}"
+  fi
+}
+
+run_script() {
+  _script="$1"
+  _action="$(basename "$_script" .sh)"
+  journal_start "$_action" "$_script"
+  if sh "$_script"; then
+    journal_ok "$_action" "$_script"
+    return 0
+  fi
+  journal_nok "$_action" "$_script"
+  return 1
+}
+
+host_short() {
+  hostname -s 2>/dev/null || hostname
+}
+
+is_primary_node() {
+  [ "$(host_short)" = "$CAST01_HOST" ]
+}
+
+is_replica_node() {
+  H="$(host_short)"
+  [ "$H" = "$CAST02_HOST" ] || [ "$H" = "$CAST03_HOST" ]
+}
+
+wait_for_tcp() {
+  _host="$1"
+  _port="$2"
+  _timeout="${3:-300}"
+  _i=0
+  while [ "$_i" -lt "$_timeout" ]; do
+    if nc -z "$_host" "$_port" 2>/dev/null; then
+      return 0
+    fi
+    _i=$((_i + 1))
+    sleep 1
+  done
+  die "timeout waiting for ${_host}:${_port}"
+}
+
+wait_for_primary_db() {
+  load_cluster_env
+  CREDS="${CAST_CLUSTER_ROOT}/credentials.txt"
+  APP_PASS="$(awk '/^\[mariadb_app\]/{f=1;next} /^\[/{f=0} f&&/^password=/{print substr($0,10); exit}' "$CREDS")"
+  _i=0
+  while [ "$_i" -lt 300 ]; do
+    if mariadb -u androidcast -p"$APP_PASS" -h"$PRIMARY_DB_HOST" -N -e 'SELECT 1' >/dev/null 2>&1; then
+      return 0
+    fi
+    _i=$((_i + 1))
+    sleep 2
+  done
+  die "primary DB not reachable at ${PRIMARY_DB_HOST}"
+}
+
+read_cred() {
+  _section="$1"
+  _key="$2"
+  _file="${CAST_CLUSTER_ROOT}/credentials.txt"
+  awk -v s="[$_section]" -v k="$_key=" '
+    $0 == s { f=1; next }
+    /^\[/ { f=0 }
+    f && index($0, k) == 1 { print substr($0, length(k)+1); exit }
+  ' "$_file"
+}
+
+ssh_node() {
+  _host="$1"
+  shift
+  ssh -o BatchMode=yes -o StrictHostKeyChecking=no "${SSH_USER}@${_host}" "$@"
+}
--- a/sim/cluster0/scripts/load-schemas.sh
+++ b/sim/cluster0/scripts/load-schemas.sh
@@ -0,0 +1,39 @@
+#!/bin/sh
+# Load project schemas on cast01 primary only.
+set -eu
+SHARED=/shared/cluster/sql
+[ "$(hostname -s)" = cast01 ] || { echo "run on cast01 only"; exit 1; }
+
+apply() {
+  f="$1"
+  [ -f "$f" ] || { echo "missing $f"; exit 1; }
+  echo "==> $f"
+  mariadb -u root < "$f"
+}
+
+apply_db() {
+  db="$1"
+  f="$2"
+  echo "==> $db < $f"
+  mariadb -u root "$db" < "$f"
+}
+
+apply "$SHARED/crashes/schema.mariadb.sql"
+
+# Migrations after base schema (003 is for legacy; base schema already includes tickets)
+for m in \
+  004_ticket_workflow.sql \
+  005_ticket_attachments.sql \
+  006_graph_sessions.sql \
+  007_remote_access.sql \
+  008_graph_sessions_indexes.sql \
+  008_auth_email_2fa.sql \
+  009_rssh_sessions.sql
+do
+  apply_db androidcast_crashes "$SHARED/crashes/migrations/$m"
+done
+
+apply "$SHARED/url_shortener/schema.mariadb.sql"
+
+mariadb -u root -e "SHOW DATABASES; SELECT COUNT(*) AS crash_tables FROM information_schema.tables WHERE table_schema='androidcast_crashes'; SELECT COUNT(*) AS url_tables FROM information_schema.tables WHERE table_schema='url_shortener';"
+echo "schemas_loaded_ok"
--- a/sim/cluster0/scripts/mariadb-primary.sh
+++ b/sim/cluster0/scripts/mariadb-primary.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+# Configure cast01 as MariaDB GTID primary. Idempotent-ish.
+set -eu
+SHARED=/shared/cluster
+CREDS="$SHARED/credentials.txt"
+REPL_USER="$(awk '/^\[mariadb_replication\]/{f=1;next} /^\[/{f=0} f&&/^user=/{print substr($0,6); exit}' "$CREDS")"
+REPL_PASS="$(awk '/^\[mariadb_replication\]/{f=1;next} /^\[/{f=0} f&&/^password=/{print substr($0,10); exit}' "$CREDS")"
+APP_USER="$(awk '/^\[mariadb_app\]/{f=1;next} /^\[/{f=0} f&&/^user=/{print substr($0,6); exit}' "$CREDS")"
+APP_PASS="$(awk '/^\[mariadb_app\]/{f=1;next} /^\[/{f=0} f&&/^password=/{print substr($0,10); exit}' "$CREDS")"
+
+[ "$(hostname -s)" = cast01 ] || { echo "run on cast01 only"; exit 1; }
+
+grep -q '^skip-networking' /etc/my.cnf.d/mariadb-server.cnf && \
+  sed -i 's/^skip-networking/#skip-networking/' /etc/my.cnf.d/mariadb-server.cnf
+
+install -D -m 644 "$SHARED/mariadb/primary.cnf" /etc/my.cnf.d/lab-cluster-primary.cnf
+rc-update add mariadb default 2>/dev/null || true
+rc-service mariadb restart
+
+mariadb -u root <<SQL
+CREATE USER IF NOT EXISTS '${REPL_USER}'@'10.7.16.%' IDENTIFIED BY '${REPL_PASS}';
+GRANT REPLICATION SLAVE, REPLICATION CLIENT, RELOAD, LOCK TABLES, SELECT, SHOW VIEW, EVENT, TRIGGER ON *.* TO '${REPL_USER}'@'10.7.16.%';
+CREATE USER IF NOT EXISTS '${APP_USER}'@'10.7.16.%' IDENTIFIED BY '${APP_PASS}';
+CREATE USER IF NOT EXISTS '${APP_USER}'@'localhost' IDENTIFIED BY '${APP_PASS}';
+GRANT ALL PRIVILEGES ON androidcast_crashes.* TO '${APP_USER}'@'10.7.16.%';
+GRANT ALL PRIVILEGES ON androidcast_crashes.* TO '${APP_USER}'@'localhost';
+GRANT ALL PRIVILEGES ON url_shortener.* TO '${APP_USER}'@'10.7.16.%';
+GRANT ALL PRIVILEGES ON url_shortener.* TO '${APP_USER}'@'localhost';
+FLUSH PRIVILEGES;
+SQL
+
+echo "primary_ok server_id=$(mariadb -u root -N -e 'SELECT @@server_id')"
+echo "gtid=$(mariadb -u root -N -e 'SELECT @@gtid_current_pos')"
--- a/sim/cluster0/scripts/mariadb-replica.sh
+++ b/sim/cluster0/scripts/mariadb-replica.sh
@@ -0,0 +1,73 @@
+#!/bin/sh
+# Bootstrap cast02 or cast03 as GTID replica of cast01.
+set -eu
+set -o pipefail
+SHARED=/shared/cluster
+CREDS="$SHARED/credentials.txt"
+[ -f "$SHARED/cluster.env" ] && . "$SHARED/cluster.env"
+HOST="$(hostname -s)"
+case "$HOST" in
+  cast02) SERVER_ID=2 ;;
+  cast03) SERVER_ID=3 ;;
+  *) echo "run on cast02 or cast03"; exit 1 ;;
+esac
+
+REPL_PASS="$(awk '/^\[mariadb_replication\]/{f=1;next} /^\[/{f=0} f&&/^password=/{print substr($0,10); exit}' "$CREDS")"
+REPL_USER="$(awk '/^\[mariadb_replication\]/{f=1;next} /^\[/{f=0} f&&/^user=/{print substr($0,6); exit}' "$CREDS")"
+APP_PASS="$(awk '/^\[mariadb_app\]/{f=1;next} /^\[/{f=0} f&&/^password=/{print substr($0,10); exit}' "$CREDS")"
+APP_USER="$(awk '/^\[mariadb_app\]/{f=1;next} /^\[/{f=0} f&&/^user=/{print substr($0,6); exit}' "$CREDS")"
+
+grep -q '^skip-networking' /etc/my.cnf.d/mariadb-server.cnf && \
+  sed -i 's/^skip-networking/#skip-networking/' /etc/my.cnf.d/mariadb-server.cnf
+
+apk add --no-cache mariadb
+install -D -m 644 "$SHARED/mariadb/replica.cnf" /etc/my.cnf.d/lab-cluster-replica.cnf
+grep -q '^server-id=' /etc/my.cnf.d/lab-cluster-replica.cnf && \
+  sed -i "s/^server-id=.*/server-id=${SERVER_ID}/" /etc/my.cnf.d/lab-cluster-replica.cnf || \
+  echo "server-id=${SERVER_ID}" >> /etc/my.cnf.d/lab-cluster-replica.cnf
+
+rc-service mariadb stop 2>/dev/null || true
+rm -rf /var/lib/mysql/*
+/etc/init.d/mariadb setup
+
+rc-update add mariadb default
+rc-service mariadb restart
+sleep 2
+
+echo "=== app DB sync from cast01 (skip mysql.* — avoids help_topic dump failures) ==="
+mariadb-dump --skip-ssl -h cast01 -u "${REPL_USER}" -p"${REPL_PASS}" \
+  --databases androidcast_crashes url_shortener \
+  --single-transaction --gtid --routines --events \
+  | mariadb -u root
+
+GTID_POS="$(mariadb --skip-ssl -h cast01 -u "${REPL_USER}" -p"${REPL_PASS}" -N -e 'SELECT @@gtid_current_pos')"
+[ -n "$GTID_POS" ] || { echo "FAIL: could not read gtid from cast01"; exit 1; }
+mariadb -u root -e "SET GLOBAL gtid_slave_pos='${GTID_POS}'"
+echo "gtid_slave_pos set to: ${GTID_POS}"
+
+mariadb -u root <<SQL
+CREATE USER IF NOT EXISTS '${APP_USER}'@'10.7.16.%' IDENTIFIED BY '${APP_PASS}';
+CREATE USER IF NOT EXISTS '${APP_USER}'@'localhost' IDENTIFIED BY '${APP_PASS}';
+GRANT ALL PRIVILEGES ON androidcast_crashes.* TO '${APP_USER}'@'10.7.16.%';
+GRANT ALL PRIVILEGES ON androidcast_crashes.* TO '${APP_USER}'@'localhost';
+GRANT ALL PRIVILEGES ON url_shortener.* TO '${APP_USER}'@'10.7.16.%';
+GRANT ALL PRIVILEGES ON url_shortener.* TO '${APP_USER}'@'localhost';
+FLUSH PRIVILEGES;
+CHANGE MASTER TO
+  MASTER_HOST='cast01',
+  MASTER_USER='${REPL_USER}',
+  MASTER_PASSWORD='${REPL_PASS}',
+  MASTER_USE_GTID=slave_pos;
+START REPLICA;
+SQL
+
+sleep 3
+mariadb -u root -e "SHOW REPLICA STATUS\G" | grep -E 'Slave_IO_Running|Slave_SQL_Running|Last_Error|Last_SQL_Error|Seconds_Behind_Master'
+mariadb -u root -e "SELECT table_schema, COUNT(*) AS tables FROM information_schema.tables WHERE table_schema IN ('androidcast_crashes','url_shortener') GROUP BY table_schema;"
+URL_N="$(mariadb -u root -N -e 'SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='"'"'url_shortener'"'"'')"
+CRASH_N="$(mariadb -u root -N -e 'SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='"'"'androidcast_crashes'"'"'')"
+EXP_CRASH="${EXPECTED_CRASHES_TABLES:-16}"
+EXP_URL="${EXPECTED_URL_SHORTENER_TABLES:-4}"
+[ "$CRASH_N" = "$EXP_CRASH" ] && [ "$URL_N" = "$EXP_URL" ] || { echo "FAIL: expected ${EXP_CRASH}+${EXP_URL} tables, got crashes=$CRASH_N url=$URL_N"; exit 1; }
+mariadb -u root -e "SHOW REPLICA STATUS\G" | grep -E 'Slave_SQL_Running: Yes' >/dev/null || { echo "FAIL: SQL thread not running"; mariadb -u root -e "SHOW REPLICA STATUS\G" | grep -E 'Last_SQL_Error'; exit 1; }
+echo "replica_ok $HOST server_id=$SERVER_ID"
--- a/sim/cluster0/scripts/verify-cluster.sh
+++ b/sim/cluster0/scripts/verify-cluster.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+# Quick health check for cast cluster.
+set -eu
+HOST="$(hostname -s)"
+FAIL=0
+echo "=== $HOST ==="
+mount | grep ' /shared ' || { echo "WARN: /shared not mounted"; FAIL=1; }
+curl -sS -o /dev/null -w "nginx=%{http_code}\n" http://127.0.0.1/ || true
+php83 -v | head -1
+
+case "$HOST" in
+  cast01)
+    mariadb -u root -e "SELECT @@server_id AS server_id, @@read_only AS read_only, @@gtid_current_pos AS gtid;"
+    mariadb -u androidcast -pcast-cluster-dev -h127.0.0.1 -e \
+      "SELECT table_schema, COUNT(*) AS tables FROM information_schema.tables WHERE table_schema IN ('androidcast_crashes','url_shortener') GROUP BY table_schema;"
+    ;;
+  cast02|cast03)
+    mariadb -u root -e "SELECT @@server_id AS server_id, @@read_only AS read_only;" 2>/dev/null || { echo "mariadb not running"; FAIL=1; }
+    RS="$(mariadb -u root -e 'SHOW REPLICA STATUS\G' 2>/dev/null || true)"
+    echo "$RS" | grep -E 'Master_Host|Slave_IO_Running|Slave_SQL_Running|Seconds_Behind_Master|Last_SQL_Error' || true
+    echo "$RS" | grep -E 'Slave_IO_Running: Yes' >/dev/null || { echo "FAIL: IO thread not running"; FAIL=1; }
+    echo "$RS" | grep -E 'Slave_SQL_Running: Yes' >/dev/null || { echo "FAIL: SQL thread not running"; FAIL=1; }
+    mariadb -u root -e \
+      "SELECT table_schema, COUNT(*) AS tables FROM information_schema.tables WHERE table_schema IN ('androidcast_crashes','url_shortener') GROUP BY table_schema;"
+    mariadb -u androidcast -pcast-cluster-dev -h127.0.0.1 -e "SELECT 1 AS local_read_ok;" 2>/dev/null || echo "WARN: local app user read failed"
+    ;;
+esac
+
+if [ "$FAIL" -eq 0 ]; then
+  echo OK
+else
+  echo FAIL
+  exit 1
+fi