ac-deploy/sim/cluster0/verify_lab_setup.sh

#!/bin/sh
# Internal cluster verification (before or after DNS). No external network required for --local.
set -eu

ROOT="$(cd "$(dirname "$0")" && pwd)"
export CAST_CLUSTER_ROOT="$ROOT"
# shellcheck source=/dev/null
. "$ROOT/scripts/lib/common.sh"
export JOURNAL_ACTOR=verify_lab
export JOURNAL_PHASE=verify

MODE=local
while [ $# -gt 0 ]; do
  case "$1" in
    --local|--local-only) MODE=local; shift ;;
    --cluster) MODE=cluster; shift ;;
    *) die "unknown arg: $1" ;;
  esac
done

FAIL=0
check_local_node() {
  H="$(host_short)"
  log "verify local $H"
  run_script "$ROOT/scripts/verify-cluster.sh" || FAIL=1

  nginx -t >/dev/null 2>&1 || { log "FAIL nginx -t"; FAIL=1; }
  rc-status nginx php-fpm83 2>/dev/null | grep -E 'nginx|php-fpm83' || true
  if is_primary_node || is_replica_node; then
    rc-service mariadb status 2>/dev/null | grep -q started || { log "FAIL mariadb not running"; FAIL=1; }
  fi

  CODE="$(curl -sS -o /dev/null -w '%{http_code}' "http://127.0.0.1${APP_BASE_PATH}/" 2>/dev/null || echo 000)"
  case "$CODE" in
    200|302) log "app_http=${CODE}" ;;
    *) log "FAIL app_http=${CODE} for ${APP_BASE_PATH}/"; FAIL=1 ;;
  esac

  ASSET="${APP_BASE_PATH}/assets/js/app.js"
  ACODE="$(curl -sS -o /dev/null -w '%{http_code}' "http://127.0.0.1${ASSET}" 2>/dev/null || echo 000)"
  case "$ACODE" in
    200) log "asset_http=${ACODE}" ;;
    *) log "WARN asset_http=${ACODE} (${ASSET})" ;;
  esac

  DIAG="$(curl -sS "http://127.0.0.1${APP_BASE_PATH}/api/diag.php" 2>/dev/null || true)"
  echo "$DIAG" | grep -q '"ok":true' || { log "FAIL diag.php"; FAIL=1; }

  LEGACY_REDIRECT="$(curl -sS -o /dev/null -w '%{http_code}' "http://127.0.0.1${APP_LEGACY_BASE_PATH:-/app/androidcast_project/crashes}/" 2>/dev/null || echo 000)"
  case "$LEGACY_REDIRECT" in
    301|308) log "legacy_redirect=${LEGACY_REDIRECT}" ;;
    *) log "WARN legacy_redirect=${LEGACY_REDIRECT} (want 301 from ${APP_LEGACY_BASE_PATH}/)" ;;
  esac

  if [ -x "$ROOT/scripts/validate-ac-repos.sh" ]; then
    sh "$ROOT/scripts/validate-ac-repos.sh" || FAIL=1
  fi

  if [ "${GITEA_ENABLE:-0}" = "1" ] && is_primary_node; then
    rc-service gitea status 2>/dev/null | grep -qi started || { log "FAIL gitea not running"; FAIL=1; }
    GCODE="$(curl -sS -o /dev/null -w '%{http_code}' http://127.0.0.1:3000/ 2>/dev/null || echo 000)"
    case "$GCODE" in
      200|302) log "gitea_direct=${GCODE}" ;;
      *) log "FAIL gitea_direct=${GCODE}"; FAIL=1 ;;
    esac
    GIT_UI="$(curl -sS -o /dev/null -w '%{http_code}' \
      "http://127.0.0.1/app/androidcast_project/git/" 2>/dev/null || echo 000)"
    case "$GIT_UI" in
      200|302) log "gitea_nginx=${GIT_UI}" ;;
      *) log "WARN gitea_nginx=${GIT_UI} (want 200/302)" ;;
    esac
    if [ -x "$ROOT/scripts/verify-gitea-mirrors.sh" ]; then
      sh "$ROOT/scripts/verify-gitea-mirrors.sh" || log "WARN gitea mirror inventory incomplete"
    fi
  fi

  CRASH_N="$(mariadb -u androidcast -p"$(read_cred mariadb_app password)" -h127.0.0.1 -N -e \
    "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='androidcast_crashes'" 2>/dev/null || echo 0)"
  [ "$CRASH_N" = "$EXPECTED_CRASHES_TABLES" ] || { log "FAIL crashes tables=$CRASH_N want $EXPECTED_CRASHES_TABLES"; FAIL=1; }
}

check_remote_node_http() {
  _host="$1"
  _ip=""
  case "$_host" in
    "$CAST01_HOST") _ip="$CAST01_IP" ;;
    "$CAST02_HOST") _ip="$CAST02_IP" ;;
    "$CAST03_HOST") _ip="$CAST03_IP" ;;
  esac
  [ -n "$_ip" ] || { log "FAIL unknown host $_host"; FAIL=1; return; }
  log "verify http $_host ($_ip)"
  _code="$(curl -sS -o /dev/null -w '%{http_code}' "http://${_ip}${APP_BASE_PATH}/" 2>/dev/null || echo 000)"
  case "$_code" in
    200|302) log "OK $_host app_http=$_code" ;;
    *) log "FAIL $_host app_http=$_code"; FAIL=1 ;;
  esac
  _diag="$(curl -sS "http://${_ip}${APP_BASE_PATH}/api/diag.php" 2>/dev/null || true)"
  echo "$_diag" | grep -q '"ok":true' || { log "FAIL $_host diag.php"; FAIL=1; }
}

check_remote_node_ssh() {
  _host="$1"
  log "verify ssh $_host"
  ssh_node "$_host" "sudo sh ${SHARED_MOUNT}/cluster/verify_lab_setup.sh --local" || FAIL=1
}

ensure_shared_mounted
journal_start verify_lab "mode=${MODE}"

if [ "$MODE" = cluster ]; then
  for H in $CAST01_HOST $CAST02_HOST $CAST03_HOST; do
    if [ "$(host_short)" = "$H" ]; then
      check_local_node
    else
      if ssh -o BatchMode=yes -o ConnectTimeout=3 "${SSH_USER}@${H}" true 2>/dev/null; then
        check_remote_node_ssh "$H"
      else
        check_remote_node_http "$H"
      fi
    fi
  done
else
  check_local_node
fi

if [ "$FAIL" -eq 0 ]; then
  journal_ok verify_lab "mode=${MODE}"
  log "verify_lab_setup_ok ${CLUSTER_NAME} mode=$MODE"
  exit 0
fi
journal_nok verify_lab "mode=${MODE}"
log "verify_lab_setup_FAIL ${CLUSTER_NAME} mode=$MODE"
exit 1