initial

sim/cluster0/scripts/lib/common.sh

@@ -0,0 +1,192 @@
+# shellcheck shell=sh
+# Shared helpers for cast cluster scripts.
+
+log() {
+  printf '[%s] %s\n' "$(date -Iseconds 2>/dev/null || date)" "$*"
+}
+
+die() {
+  log "ERROR: $*"
+  if [ -n "${JOURNAL_LAST_ACTION:-}" ]; then
+    journal_record NOK "$JOURNAL_LAST_ACTION" "$*"
+  fi
+  exit 1
+}
+
+cluster_root() {
+  if [ -n "${CAST_CLUSTER_ROOT:-}" ]; then
+    printf '%s\n' "$CAST_CLUSTER_ROOT"
+    return
+  fi
+  if [ -f /shared/cluster/cluster.env ]; then
+    printf '/shared/cluster\n'
+    return
+  fi
+  if [ -f "$(dirname "$0")/../cluster.env" ]; then
+    cd "$(dirname "$0")/.." && pwd
+    return
+  fi
+  die "cannot locate cluster root (is /shared mounted?)"
+}
+
+load_cluster_env() {
+  ROOT="$(cluster_root)"
+  ENV_FILE="${1:-$ROOT/cluster.env}"
+  [ -f "$ENV_FILE" ] || die "missing $ENV_FILE"
+  # shellcheck disable=SC1090
+  . "$ENV_FILE"
+  CAST_CLUSTER_ROOT="$ROOT"
+  FE_NFS="${FE_NFS_SERVER}:${FE_NFS_EXPORT_PATH}"
+  FSTAB_NFS_LINE="${FE_NFS} ${SHARED_MOUNT} nfs rw,_netdev,nofail,noatime,nodiratime 0 0"
+  ALL_CAST_HOSTS="${CAST01_HOST} ${CAST02_HOST} ${CAST03_HOST}"
+  JOURNAL_DIR="${SHARED_MOUNT}/journal"
+  JOURNAL_TSV="${JOURNAL_DIR}/${CLUSTER_NAME}.journal.tsv"
+  JOURNAL_SUMMARY="${JOURNAL_DIR}/${CLUSTER_NAME}.summary.md"
+}
+
+journal_init() {
+  load_cluster_env 2>/dev/null || return 0
+  mkdir -p "$JOURNAL_DIR"
+  if [ ! -f "$JOURNAL_TSV" ]; then
+    printf 'timestamp_utc\thost\tactor\tphase\taction\tstatus\tdetail\n' > "$JOURNAL_TSV"
+  fi
+}
+
+journal_record() {
+  _status="$1"
+  _action="$2"
+  _detail="${3:-}"
+  _phase="${JOURNAL_PHASE:-}"
+  journal_init
+  _ts="$(date -u -Iseconds 2>/dev/null || date -u +%Y-%m-%dT%H:%M:%SZ)"
+  _host="$(host_short 2>/dev/null || echo unknown)"
+  _actor="${JOURNAL_ACTOR:-script}"
+  _line="${_ts}	${_host}	${_actor}	${_phase}	${_action}	${_status}	${_detail}"
+  printf '%s\n' "$_line" >> "$JOURNAL_TSV"
+  log "journal ${_status} ${_action}${_detail:+ — ${_detail}}"
+  if [ -x "${CAST_CLUSTER_ROOT}/scripts/journal-rebuild-summary.sh" ] 2>/dev/null; then
+    sh "${CAST_CLUSTER_ROOT}/scripts/journal-rebuild-summary.sh" 2>/dev/null || true
+  elif [ -f "${CAST_CLUSTER_ROOT}/scripts/journal-rebuild-summary.sh" ]; then
+    sh "${CAST_CLUSTER_ROOT}/scripts/journal-rebuild-summary.sh" 2>/dev/null || true
+  fi
+}
+
+journal_start() {
+  JOURNAL_LAST_ACTION="$1"
+  journal_record START "$1" "${2:-}"
+}
+
+journal_ok() {
+  journal_record OK "$1" "${2:-}"
+  JOURNAL_LAST_ACTION=""
+}
+
+journal_nok() {
+  journal_record NOK "$1" "${2:-}"
+  JOURNAL_LAST_ACTION=""
+}
+
+journal_warn() {
+  journal_record WARN "$1" "${2:-}"
+}
+
+ensure_shared_mounted() {
+  load_cluster_env
+  if mount | grep -q " on ${SHARED_MOUNT} "; then
+    journal_init
+    return 0
+  fi
+  if [ -f /etc/cast-cluster.env ]; then
+    # shellcheck disable=SC1091
+    . /etc/cast-cluster.env
+    FE_NFS="${FE_NFS_SERVER}:${FE_NFS_EXPORT_PATH}"
+    FSTAB_NFS_LINE="${FE_NFS} ${SHARED_MOUNT} nfs rw,_netdev,nofail,noatime,nodiratime 0 0"
+  fi
+  log "mounting ${SHARED_MOUNT} from ${FE_NFS}"
+  journal_start nfs_mount "from ${FE_NFS}"
+  mkdir -p "$SHARED_MOUNT"
+  grep -v "${FE_NFS_EXPORT_PATH}" /etc/fstab > /var/tmp/cast-fstab.new 2>/dev/null || cp /etc/fstab /var/tmp/cast-fstab.new
+  echo "$FSTAB_NFS_LINE" >> /var/tmp/cast-fstab.new
+  mv /var/tmp/cast-fstab.new /etc/fstab
+  rc-update add nfsmount boot 2>/dev/null || true
+  mount "$SHARED_MOUNT" 2>/dev/null || mount -a
+  if mount | grep -q " on ${SHARED_MOUNT} "; then
+    load_cluster_env
+    journal_ok nfs_mount "${SHARED_MOUNT}"
+  else
+    journal_nok nfs_mount "mount failed"
+    die "NFS mount failed: ${SHARED_MOUNT}"
+  fi
+}
+
+run_script() {
+  _script="$1"
+  _action="$(basename "$_script" .sh)"
+  journal_start "$_action" "$_script"
+  if sh "$_script"; then
+    journal_ok "$_action" "$_script"
+    return 0
+  fi
+  journal_nok "$_action" "$_script"
+  return 1
+}
+
+host_short() {
+  hostname -s 2>/dev/null || hostname
+}
+
+is_primary_node() {
+  [ "$(host_short)" = "$CAST01_HOST" ]
+}
+
+is_replica_node() {
+  H="$(host_short)"
+  [ "$H" = "$CAST02_HOST" ] || [ "$H" = "$CAST03_HOST" ]
+}
+
+wait_for_tcp() {
+  _host="$1"
+  _port="$2"
+  _timeout="${3:-300}"
+  _i=0
+  while [ "$_i" -lt "$_timeout" ]; do
+    if nc -z "$_host" "$_port" 2>/dev/null; then
+      return 0
+    fi
+    _i=$((_i + 1))
+    sleep 1
+  done
+  die "timeout waiting for ${_host}:${_port}"
+}
+
+wait_for_primary_db() {
+  load_cluster_env
+  CREDS="${CAST_CLUSTER_ROOT}/credentials.txt"
+  APP_PASS="$(awk '/^\[mariadb_app\]/{f=1;next} /^\[/{f=0} f&&/^password=/{print substr($0,10); exit}' "$CREDS")"
+  _i=0
+  while [ "$_i" -lt 300 ]; do
+    if mariadb -u androidcast -p"$APP_PASS" -h"$PRIMARY_DB_HOST" -N -e 'SELECT 1' >/dev/null 2>&1; then
+      return 0
+    fi
+    _i=$((_i + 1))
+    sleep 2
+  done
+  die "primary DB not reachable at ${PRIMARY_DB_HOST}"
+}
+
+read_cred() {
+  _section="$1"
+  _key="$2"
+  _file="${CAST_CLUSTER_ROOT}/credentials.txt"
+  awk -v s="[$_section]" -v k="$_key=" '
+    $0 == s { f=1; next }
+    /^\[/ { f=0 }
+    f && index($0, k) == 1 { print substr($0, length(k)+1); exit }
+  ' "$_file"
+}
+
+ssh_node() {
+  _host="$1"
+  shift
+  ssh -o BatchMode=yes -o StrictHostKeyChecking=no "${SSH_USER}@${_host}" "$@"
+}