From 89a03650382d015f6f0e9af792fd395b1239c7fe Mon Sep 17 00:00:00 2001 From: Anton Afanasyeu Date: Wed, 10 Jun 2026 22:03:31 +0200 Subject: [PATCH] be sync --- .../backend/scripts/gitea/README.md | 56 ++++ .../fe-android_cast-post-receive.example | 32 +++ .../scripts/gitea/gitea_migrate_project.sh | 246 ++++++++++++++++++ .../backend/scripts/gitea/gitea_mirror_lib.sh | 31 +++ .../scripts/gitea/gitea_submodule_lib.sh | 116 +++++++++ .../scripts/gitea/gitea_sync_mirror.sh | 63 +++++ 6 files changed, 544 insertions(+) create mode 100644 examples/crash_reporter/backend/scripts/gitea/fe-android_cast-post-receive.example create mode 100755 examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh create mode 100644 examples/crash_reporter/backend/scripts/gitea/gitea_submodule_lib.sh create mode 100755 examples/crash_reporter/backend/scripts/gitea/gitea_sync_mirror.sh diff --git a/examples/crash_reporter/backend/scripts/gitea/README.md b/examples/crash_reporter/backend/scripts/gitea/README.md index 1bf618c..000e913 100644 --- a/examples/crash_reporter/backend/scripts/gitea/README.md +++ b/examples/crash_reporter/backend/scripts/gitea/README.md @@ -154,10 +154,66 @@ DEFAULT_MIRROR_INTERVAL = 10m MIRROR_UPDATE_INTERVAL = 10m ``` +## Full project migration (main + external submodules) + +**`gitea_migrate_project.sh`** — one script for BE: + +1. Pull-mirror **`android_cast`** from FE git (`git://10.7.0.10/android_cast`) with a **short interval** (default `1m`). +2. Read **external** submodules from a checkout (`.gitmodules` + **pinned SHA** at `HEAD` / `--ref`). +3. Create read-only pull mirrors for each upstream (libvpx, opus, speex, …). +4. Verify the **same commit digest** the superproject pins is reachable on each Gitea mirror. + +Canonical `.gitmodules` stays on upstream URLs — no remote edit. + +```bash +cd /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast +chmod +x examples/crash_reporter/backend/scripts/gitea/*.sh + +sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh . + +# Re-run submodules only (main mirror already exists): +sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh . --skip-main + +# Pin check at a tag/branch: +GITEA_REF=next sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh . +``` + +### Fast sync for local `android_cast` mirror only + +Gitea **default minimum** pull interval is **10m**. To use **1m** for the FE git mirror: + +```ini +[mirror] +MIN_INTERVAL = 1m + +[cron.update_mirrors] +SCHEDULE = @every 1m +RUN_AT_START = true +``` + +Restart Gitea. The migrate script sets `admin/android_cast` mirror interval to `GITEA_MAIN_MIRROR_INTERVAL` (default `1m`). External submodule mirrors default to `24h` (`GITEA_SUBMODULE_MIRROR_INTERVAL`). + +**Near-immediate sync on each FE push** (better than polling): + +| Method | Command | +|--------|---------| +| BE cron every minute | `* * * * * root /path/to/gitea_sync_mirror.sh android_cast` | +| FE `post-receive` hook | Copy `fe-android_cast-post-receive.example` → FE bare repo `hooks/post-receive`; set `GITEA_SYNC_CMD` (ssh to BE or curl `mirror-sync` API) | +| Manual | `sudo ./gitea_sync_mirror.sh android_cast` | + +```bash +sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_sync_mirror.sh android_cast +sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_sync_mirror.sh --all +``` + +There is no built-in “push mirror from FE git daemon → Gitea” in stock git; **pull mirror + trigger** (`mirror-sync` API or hook) is the usual integration when git lives on FE and Gitea on BE. + ## Submodule mirrors (libvpx, opus, speex, …) Gitea does not “nest” submodules inside one repo — each submodule is its **own pull mirror** (`admin/libvpx`, `admin/opus`, …). That works **without** changing committed `.gitmodules`. +Prefer **`gitea_migrate_project.sh`** for the full flow; or submodule-only: + **1. Allow upstream hosts** in `app.ini` (see `[migrations]` block above — includes GitHub, googlesource, zx2c4). **2. Run on BE (as root):** diff --git a/examples/crash_reporter/backend/scripts/gitea/fe-android_cast-post-receive.example b/examples/crash_reporter/backend/scripts/gitea/fe-android_cast-post-receive.example new file mode 100644 index 0000000..a0bdd7b --- /dev/null +++ b/examples/crash_reporter/backend/scripts/gitea/fe-android_cast-post-receive.example @@ -0,0 +1,32 @@ +#!/bin/sh +# FE bare repo hook: android_cast.git/hooks/post-receive +# After push to git://10.7.0.10/android_cast, trigger immediate Gitea pull sync on BE. +# +# Install on Gentoo FE (adjust paths): +# cp fe-android_cast-post-receive.example /var/git/android_cast.git/hooks/post-receive +# chmod +x /var/git/android_cast.git/hooks/post-receive +# +# Pick ONE trigger method below (uncomment). + +# --- A) SSH to BE (recommended if FE has alpine-be key) --- +# GITEA_SYNC_CMD='ssh alpine-be sudo /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/crash_reporter/backend/scripts/gitea/gitea_sync_mirror.sh android_cast' + +# --- B) curl Gitea API on BE loopback via FE→BE nginx/internal URL --- +# Create a long-lived token in Gitea UI (admin, scope: write repository). +# GITEA_SYNC_CMD='curl -fsS -X POST -H "Authorization: token YOUR_TOKEN" http://artc0.intra.raptor.org:3000/api/v1/repos/admin/android_cast/mirror-sync' + +# --- C) curl public apps URL (only if you add a protected internal endpoint) --- +# GITEA_SYNC_CMD='curl -fsS -X POST -H "Authorization: Bearer ..." https://apps.f0xx.org/.../gitea_mirror_sync' + +: "${GITEA_SYNC_CMD:=}" +while read -r _old _new _ref; do + [ "$_new" = "0000000000000000000000000000000000000000" ] && continue + case "$_ref" in + refs/heads/next|refs/heads/master) ;; + *) continue ;; + esac + if [ -n "$GITEA_SYNC_CMD" ]; then + sh -c "$GITEA_SYNC_CMD" /dev/null 2>&1 & + fi +done +exit 0 diff --git a/examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh b/examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh new file mode 100755 index 0000000..a8c8b56 --- /dev/null +++ b/examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh @@ -0,0 +1,246 @@ +#!/bin/ash +# +# One-shot Gitea migration for android_cast: +# 1) Pull-mirror main repo from FE git (git://10.7.0.10/android_cast) — fast interval +# 2) Discover external git submodules (.gitmodules + pinned SHA at REF) +# 3) Pull-mirror each upstream as read-only repo; verify project commit exists +# +# Canonical .gitmodules stays unchanged (upstream URLs). Use gitea_submodule_local_rewrite.sh +# locally if clones should fetch submodules via Gitea. +# +# Usage (Alpine BE, as root): +# sudo ./gitea_migrate_project.sh +# sudo ./gitea_migrate_project.sh /var/www/.../androidcast_project/android_cast +# GITEA_MIRROR_URL=git://10.7.0.10/android_cast GITEA_MAIN_MIRROR_INTERVAL=1m sudo ./gitea_migrate_project.sh +# +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=gitea_ini.sh +. "$SCRIPT_DIR/gitea_ini.sh" +# shellcheck source=gitea_mirror_lib.sh +. "$SCRIPT_DIR/gitea_mirror_lib.sh" +# shellcheck source=gitea_submodule_lib.sh +. "$SCRIPT_DIR/gitea_submodule_lib.sh" + +GITEA_OWNER="${GITEA_OWNER:-admin}" +GITEA_REPO="${GITEA_REPO:-android_cast}" +GITEA_MIRROR_URL="${GITEA_MIRROR_URL:-git://10.7.0.10/android_cast}" +GITEA_MAIN_MIRROR_INTERVAL="${GITEA_MAIN_MIRROR_INTERVAL:-1m}" +GITEA_SUBMODULE_MIRROR_INTERVAL="${GITEA_SUBMODULE_MIRROR_INTERVAL:-24h}" +GITEA_PRIVATE="${GITEA_PRIVATE:-false}" +GITEA_REF="${GITEA_REF:-HEAD}" +GITEA_SKIP_MAIN="${GITEA_SKIP_MAIN:-0}" +GITEA_SKIP_SUBMODULES="${GITEA_SKIP_SUBMODULES:-0}" +WORK_DIR="${WORK_DIR:-/tmp/gitea-migrate-$$}" + +log() { printf '[gitea-migrate] %s\n' "$*"; } +warn() { printf '[gitea-migrate] WARN: %s\n' "$*" >&2; } +die() { printf '[gitea-migrate] ERROR: %s\n' "$*" >&2; exit 1; } + +resolve_repo_dir() { + _arg="${1:-}" + if [ -n "$_arg" ]; then + [ -d "$_arg" ] || die "not a directory: $_arg" + printf '%s' "$(cd "$_arg" && pwd)" + return 0 + fi + if [ -d "./.git" ] || [ -f "./.git" ]; then + pwd + return 0 + fi + die "pass android_cast checkout path (needs .gitmodules and git history)" +} + +migrate_main_repo() { + log "main mirror: $GITEA_OWNER/$GITEA_REPO <- $GITEA_MIRROR_URL (interval $GITEA_MAIN_MIRROR_INTERVAL)" + if git ls-remote "$GITEA_MIRROR_URL" HEAD >/dev/null 2>&1; then + log " upstream reachable" + else + warn " upstream not reachable from BE — check FE git daemon / [migrations] ALLOW_LOCALNETWORKS" + fi + + if gitea_mirror_pull_repo "$GITEA_MIRROR_URL" "$GITEA_REPO" "$GITEA_OWNER" \ + "$GITEA_MAIN_MIRROR_INTERVAL" "$WORK_DIR"; then + gitea_mirror_set_interval_api "$GITEA_OWNER" "$GITEA_REPO" "$GITEA_MAIN_MIRROR_INTERVAL" \ + 2>/dev/null || true + gitea_sync_mirror_api "$GITEA_OWNER" "$GITEA_REPO" || true + log " OK -> $(gitea_public_repo_url_for "$GITEA_OWNER" "$GITEA_REPO")" + return 0 + fi + warn " main mirror FAILED" + return 1 +} + +migrate_submodule() { + _line="$1" + _repo="${_line%%|*}" + _rest="${_line#*|}" + _upstream="${_rest%%|*}" + _rest2="${_rest#*|}" + _path="${_rest2%%|*}" + _rest3="${_rest2#*|}" + _sha="${_rest3%%|*}" + _branch="${_rest3#*|}" + case "$_branch" in + "$_sha") _branch="" ;; + esac + + log "submodule $_path ($GITEA_OWNER/$_repo)" + log " upstream: $_upstream" + log " pinned @ $GITEA_REF: $_sha" + + if git ls-remote "$_upstream" "$_sha" >/dev/null 2>&1; then + log " commit on upstream" + else + warn " commit $_sha not on upstream (mirror may still fetch via branch tip)" + fi + + if ! gitea_mirror_pull_repo "$_upstream" "$_repo" "$GITEA_OWNER" \ + "$GITEA_SUBMODULE_MIRROR_INTERVAL" "$WORK_DIR"; then + warn " mirror create/sync FAILED for $_repo" + return 1 + fi + + gitea_mirror_set_interval_api "$GITEA_OWNER" "$_repo" "$GITEA_SUBMODULE_MIRROR_INTERVAL" \ + 2>/dev/null || true + gitea_sync_mirror_api "$GITEA_OWNER" "$_repo" || true + + if gitea_mirror_has_commit "$GITEA_OWNER" "$_repo" "$_sha"; then + log " OK pinned commit present -> $(gitea_public_repo_url_for "$GITEA_OWNER" "$_repo")" + return 0 + fi + + warn " mirror ok but pinned $_sha not visible yet — retry: $SCRIPT_DIR/gitea_sync_mirror.sh $_repo" + return 2 +} + +print_fast_sync_guide() { + _sync_sh="$SCRIPT_DIR/gitea_sync_mirror.sh" + _hook="$SCRIPT_DIR/fe-android_cast-post-receive.example" + cat </dev/null 2>&1; then + rc-service gitea status 2>/dev/null | grep -qi started || rc-service gitea start 2>/dev/null || true +fi + +mkdir -p "$WORK_DIR" +trap 'rm -rf "$WORK_DIR"' EXIT INT HUP TERM + +log "ini=$GITEA_APP_INI owner=$GITEA_OWNER ref=$GITEA_REF" + +_main_ok=0 +_main_fail=0 +_sub_ok=0 +_sub_warn=0 +_sub_fail=0 + +if [ "$GITEA_SKIP_MAIN" -eq 0 ]; then + if migrate_main_repo; then + _main_ok=1 + else + _main_fail=1 + fi +else + log "skipping main mirror (--skip-main)" +fi + +if [ "$GITEA_SKIP_SUBMODULES" -eq 0 ]; then + _repo_dir="$(resolve_repo_dir "$_repo_arg")" + log "discovering external submodules in $_repo_dir" + _sub_list="$WORK_DIR/submodules.list" + : > "$_sub_list" + gitea_submodule_discover_external "$_repo_dir" "$GITEA_REF" > "$_sub_list" || true + if [ ! -s "$_sub_list" ]; then + warn "no external submodules at $GITEA_REF (empty .gitmodules or only local URLs)" + else + while IFS= read -r _line; do + [ -n "$_line" ] || continue + gitea_submodule_entry_valid "$_line" || continue + _rc=0 + migrate_submodule "$_line" || _rc=$? + case "$_rc" in + 0) _sub_ok=$((_sub_ok + 1)) ;; + 2) _sub_warn=$((_sub_warn + 1)) ;; + *) _sub_fail=$((_sub_fail + 1)) ;; + esac + done < "$_sub_list" + fi +else + log "skipping submodule mirrors (--skip-submodules)" +fi + +log "summary: main ok=$_main_ok fail=$_main_fail | submodules ok=$_sub_ok warn=$_sub_warn fail=$_sub_fail" +print_fast_sync_guide + +[ "$_main_fail" -eq 0 ] && [ "$_sub_fail" -eq 0 ] diff --git a/examples/crash_reporter/backend/scripts/gitea/gitea_mirror_lib.sh b/examples/crash_reporter/backend/scripts/gitea/gitea_mirror_lib.sh index 1d945ee..a23099c 100755 --- a/examples/crash_reporter/backend/scripts/gitea/gitea_mirror_lib.sh +++ b/examples/crash_reporter/backend/scripts/gitea/gitea_mirror_lib.sh @@ -149,6 +149,36 @@ gitea_sync_mirror_api() { -H "Authorization: token $_token" | grep -q '^20' } +gitea_mirror_set_interval_api() { + _owner="$1" + _repo="$2" + _interval="$3" + command -v curl >/dev/null 2>&1 || return 1 + _token="${GITEA_ADMIN_TOKEN:-}" + [ -n "$_token" ] || _token="$(gitea_resolve_push_token)" || return 1 + _api="$(gitea_api_base)" + _payload="{\"mirror_interval\":\"$_interval\"}" + _http="$(curl -sS -o /tmp/gitea-mirror-interval-$$.json -w '%{http_code}' \ + -X PATCH "$_api/repos/$_owner/$_repo" \ + -H "Authorization: token $_token" \ + -H "Content-Type: application/json" \ + -d "$_payload")" + case "$_http" in + 200|204) return 0 ;; + *) return 1 ;; + esac +} + +# True if commit SHA is reachable on the mirror (after sync). +gitea_mirror_has_commit() { + _owner="$1" + _repo="$2" + _sha="$3" + command -v git >/dev/null 2>&1 || return 1 + _url="$(gitea_local_git_url_for "$_owner" "$_repo")" + git ls-remote "$_url" "$_sha" 2>/dev/null | grep -q "$_sha" +} + # Mirror one upstream into Gitea (API migrate, else git push). Returns 0 on success. gitea_mirror_pull_repo() { _clone_url="$1" @@ -162,6 +192,7 @@ gitea_mirror_pull_repo() { && return 0 _api_rc=$? if [ "$_api_rc" -eq 2 ]; then + gitea_mirror_set_interval_api "$_owner" "$_repo" "$_interval" 2>/dev/null || true gitea_sync_mirror_api "$_owner" "$_repo" && return 0 return 0 fi diff --git a/examples/crash_reporter/backend/scripts/gitea/gitea_submodule_lib.sh b/examples/crash_reporter/backend/scripts/gitea/gitea_submodule_lib.sh new file mode 100644 index 0000000..c24fa02 --- /dev/null +++ b/examples/crash_reporter/backend/scripts/gitea/gitea_submodule_lib.sh @@ -0,0 +1,116 @@ +#!/bin/ash +# Discover git submodules (upstream URL + pinned SHA) from a checkout. +# shellcheck shell=sh + +# Space-separated host/prefix patterns treated as "ours" (skipped for external mirrors). +GITEA_LOCAL_URL_PATTERNS="${GITEA_LOCAL_URL_PATTERNS:-10.7.0.10 10.7.16. apps.f0xx.org f0xx.org 127.0.0.1 localhost}" + +gitea_submodule_is_local_url() { + _url="$1" + case "$_url" in + /*|./*|../*) return 0 ;; + esac + for _pat in $GITEA_LOCAL_URL_PATTERNS; do + case "$_url" in + *"$_pat"*) return 0 ;; + esac + done + return 1 +} + +gitea_submodule_is_external_url() { + gitea_submodule_is_local_url "$1" && return 1 + return 0 +} + +# .gitmodules -> repo|url|path|branch (repo = basename of path). stdout only. +gitea_submodule_parse_gitmodules() { + _file="$1" + [ -r "$_file" ] || return 1 + awk ' + function trim(s) { + sub(/^[ \t]+/, "", s); sub(/[ \t]+$/, "", s) + return s + } + /^[ \t]*path[ \t]*=/ { + p = $0; sub(/^[^=]*=/, "", p); path = trim(p) + n = path; sub(".*/", "", n) + next + } + /^[ \t]*url[ \t]*=/ { + u = $0; sub(/^[^=]*=/, "", u); url = trim(u) + next + } + /^[ \t]*branch[ \t]*=/ { + b = $0; sub(/^[^=]*=/, "", b); branch = trim(b) + next + } + /^\[/ { + if (path != "" && url != "" && n != "") { + if (branch != "") print n "|" url "|" path "|" branch + else print n "|" url "|" path + } + path = ""; url = ""; branch = ""; n = "" + } + END { + if (path != "" && url != "" && n != "") { + if (branch != "") print n "|" url "|" path "|" branch + else print n "|" url "|" path + } + } + ' "$_file" +} + +# repo|url|path|sha[|branch] for each external submodule with a gitlink at REF. +gitea_submodule_discover_external() { + _repo_dir="$1" + _ref="${2:-HEAD}" + _gitmodules="$_repo_dir/.gitmodules" + + command -v git >/dev/null 2>&1 || return 1 + git -C "$_repo_dir" rev-parse --verify "$_ref" >/dev/null 2>&1 || return 1 + [ -r "$_gitmodules" ] || return 1 + + gitea_submodule_parse_gitmodules "$_gitmodules" | while IFS= read -r _line; do + [ -n "$_line" ] || continue + _repo="${_line%%|*}" + _rest="${_line#*|}" + _url="${_rest%%|*}" + _rest2="${_rest#*|}" + _path="${_rest2%%|*}" + _branch="${_rest2#*|}" + case "$_branch" in + "$_path") _branch="" ;; + esac + + gitea_submodule_is_external_url "$_url" || continue + + _sha="$(git -C "$_repo_dir" rev-parse "$_ref:$_path" 2>/dev/null)" || continue + case "$_sha" in + ??000000000000000000000000000000000000) continue ;; + esac + + if [ -n "$_branch" ]; then + printf '%s|%s|%s|%s|%s\n' "$_repo" "$_url" "$_path" "$_sha" "$_branch" + else + printf '%s|%s|%s|%s\n' "$_repo" "$_url" "$_path" "$_sha" + fi + done +} + +gitea_submodule_entry_valid() { + case "$1" in + *'|'*) ;; + *) return 1 ;; + esac + _repo="${1%%|*}" + _rest="${1#*|}" + _url="${_rest%%|*}" + _rest2="${_rest#*|}" + _path="${_rest2%%|*}" + [ -n "$_repo" ] && [ -n "$_url" ] && [ -n "$_path" ] || return 1 + case "$_repo" in + *' '*|*'['*|*']'*) return 1 ;; + esac + return 0 +} diff --git a/examples/crash_reporter/backend/scripts/gitea/gitea_sync_mirror.sh b/examples/crash_reporter/backend/scripts/gitea/gitea_sync_mirror.sh new file mode 100755 index 0000000..05a6502 --- /dev/null +++ b/examples/crash_reporter/backend/scripts/gitea/gitea_sync_mirror.sh @@ -0,0 +1,63 @@ +#!/bin/ash +# +# Trigger Gitea pull-mirror sync (one repo or all). For BE cron / FE post-receive hook. +# +# Usage: +# sudo ./gitea_sync_mirror.sh android_cast +# sudo ./gitea_sync_mirror.sh android_cast libvpx opus +# sudo ./gitea_sync_mirror.sh --all +# +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=gitea_ini.sh +. "$SCRIPT_DIR/gitea_ini.sh" +# shellcheck source=gitea_mirror_lib.sh +. "$SCRIPT_DIR/gitea_mirror_lib.sh" + +GITEA_OWNER="${GITEA_OWNER:-admin}" + +log() { printf '[gitea-sync] %s\n' "$*"; } +warn() { printf '[gitea-sync] WARN: %s\n' "$*" >&2; } +die() { printf '[gitea-sync] ERROR: %s\n' "$*" >&2; exit 1; } + +gitea_load_config || exit 1 +gitea_ensure_cli_access || exit 1 + +_sync_all=0 +_repos="" + +while [ $# -gt 0 ]; do + case "$1" in + -h|--help) + sed -n '2,10p' "$0" + exit 0 + ;; + --all) _sync_all=1; shift ;; + -*) die "unknown option: $1" ;; + *) _repos="$_repos $1"; shift ;; + esac +done + +if [ "$_sync_all" -eq 1 ]; then + log "sync all mirrors via gitea admin repo-sync-mirrors" + gitea_cli "admin" "repo-sync-mirrors" + exit $? +fi + +if [ -z "$(echo "$_repos" | tr -d ' ')" ]; then + _repos="android_cast" +fi + +_fail=0 +for _repo in $_repos; do + log "mirror-sync $GITEA_OWNER/$_repo" + if gitea_sync_mirror_api "$GITEA_OWNER" "$_repo"; then + log " OK" + else + warn " FAILED" + _fail=1 + fi +done + +exit "$_fail"