1
0
mirror of git://f0xx.org/android_cast synced 2026-07-29 05:58:14 +03:00
This commit is contained in:
Anton Afanasyeu
2026-06-10 22:03:31 +02:00
parent 8dae3b0615
commit 89a0365038
6 changed files with 544 additions and 0 deletions

View File

@@ -154,10 +154,66 @@ DEFAULT_MIRROR_INTERVAL = 10m
MIRROR_UPDATE_INTERVAL = 10m MIRROR_UPDATE_INTERVAL = 10m
``` ```
## Full project migration (main + external submodules)
**`gitea_migrate_project.sh`** — one script for BE:
1. Pull-mirror **`android_cast`** from FE git (`git://10.7.0.10/android_cast`) with a **short interval** (default `1m`).
2. Read **external** submodules from a checkout (`.gitmodules` + **pinned SHA** at `HEAD` / `--ref`).
3. Create read-only pull mirrors for each upstream (libvpx, opus, speex, …).
4. Verify the **same commit digest** the superproject pins is reachable on each Gitea mirror.
Canonical `.gitmodules` stays on upstream URLs — no remote edit.
```bash
cd /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast
chmod +x examples/crash_reporter/backend/scripts/gitea/*.sh
sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh .
# Re-run submodules only (main mirror already exists):
sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh . --skip-main
# Pin check at a tag/branch:
GITEA_REF=next sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh .
```
### Fast sync for local `android_cast` mirror only
Gitea **default minimum** pull interval is **10m**. To use **1m** for the FE git mirror:
```ini
[mirror]
MIN_INTERVAL = 1m
[cron.update_mirrors]
SCHEDULE = @every 1m
RUN_AT_START = true
```
Restart Gitea. The migrate script sets `admin/android_cast` mirror interval to `GITEA_MAIN_MIRROR_INTERVAL` (default `1m`). External submodule mirrors default to `24h` (`GITEA_SUBMODULE_MIRROR_INTERVAL`).
**Near-immediate sync on each FE push** (better than polling):
| Method | Command |
|--------|---------|
| BE cron every minute | `* * * * * root /path/to/gitea_sync_mirror.sh android_cast` |
| FE `post-receive` hook | Copy `fe-android_cast-post-receive.example` → FE bare repo `hooks/post-receive`; set `GITEA_SYNC_CMD` (ssh to BE or curl `mirror-sync` API) |
| Manual | `sudo ./gitea_sync_mirror.sh android_cast` |
```bash
sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_sync_mirror.sh android_cast
sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_sync_mirror.sh --all
```
There is no built-in “push mirror from FE git daemon → Gitea” in stock git; **pull mirror + trigger** (`mirror-sync` API or hook) is the usual integration when git lives on FE and Gitea on BE.
## Submodule mirrors (libvpx, opus, speex, …) ## Submodule mirrors (libvpx, opus, speex, …)
Gitea does not “nest” submodules inside one repo — each submodule is its **own pull mirror** (`admin/libvpx`, `admin/opus`, …). That works **without** changing committed `.gitmodules`. Gitea does not “nest” submodules inside one repo — each submodule is its **own pull mirror** (`admin/libvpx`, `admin/opus`, …). That works **without** changing committed `.gitmodules`.
Prefer **`gitea_migrate_project.sh`** for the full flow; or submodule-only:
**1. Allow upstream hosts** in `app.ini` (see `[migrations]` block above — includes GitHub, googlesource, zx2c4). **1. Allow upstream hosts** in `app.ini` (see `[migrations]` block above — includes GitHub, googlesource, zx2c4).
**2. Run on BE (as root):** **2. Run on BE (as root):**

View File

@@ -0,0 +1,32 @@
#!/bin/sh
# FE bare repo hook: android_cast.git/hooks/post-receive
# After push to git://10.7.0.10/android_cast, trigger immediate Gitea pull sync on BE.
#
# Install on Gentoo FE (adjust paths):
# cp fe-android_cast-post-receive.example /var/git/android_cast.git/hooks/post-receive
# chmod +x /var/git/android_cast.git/hooks/post-receive
#
# Pick ONE trigger method below (uncomment).
# --- A) SSH to BE (recommended if FE has alpine-be key) ---
# GITEA_SYNC_CMD='ssh alpine-be sudo /var/www/localhost/htdocs/apps/app/androidcast_project/android_cast/examples/crash_reporter/backend/scripts/gitea/gitea_sync_mirror.sh android_cast'
# --- B) curl Gitea API on BE loopback via FE→BE nginx/internal URL ---
# Create a long-lived token in Gitea UI (admin, scope: write repository).
# GITEA_SYNC_CMD='curl -fsS -X POST -H "Authorization: token YOUR_TOKEN" http://artc0.intra.raptor.org:3000/api/v1/repos/admin/android_cast/mirror-sync'
# --- C) curl public apps URL (only if you add a protected internal endpoint) ---
# GITEA_SYNC_CMD='curl -fsS -X POST -H "Authorization: Bearer ..." https://apps.f0xx.org/.../gitea_mirror_sync'
: "${GITEA_SYNC_CMD:=}"
while read -r _old _new _ref; do
[ "$_new" = "0000000000000000000000000000000000000000" ] && continue
case "$_ref" in
refs/heads/next|refs/heads/master) ;;
*) continue ;;
esac
if [ -n "$GITEA_SYNC_CMD" ]; then
sh -c "$GITEA_SYNC_CMD" </dev/null >/dev/null 2>&1 &
fi
done
exit 0

View File

@@ -0,0 +1,246 @@
#!/bin/ash
#
# One-shot Gitea migration for android_cast:
# 1) Pull-mirror main repo from FE git (git://10.7.0.10/android_cast) — fast interval
# 2) Discover external git submodules (.gitmodules + pinned SHA at REF)
# 3) Pull-mirror each upstream as read-only repo; verify project commit exists
#
# Canonical .gitmodules stays unchanged (upstream URLs). Use gitea_submodule_local_rewrite.sh
# locally if clones should fetch submodules via Gitea.
#
# Usage (Alpine BE, as root):
# sudo ./gitea_migrate_project.sh
# sudo ./gitea_migrate_project.sh /var/www/.../androidcast_project/android_cast
# GITEA_MIRROR_URL=git://10.7.0.10/android_cast GITEA_MAIN_MIRROR_INTERVAL=1m sudo ./gitea_migrate_project.sh
#
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
# shellcheck source=gitea_ini.sh
. "$SCRIPT_DIR/gitea_ini.sh"
# shellcheck source=gitea_mirror_lib.sh
. "$SCRIPT_DIR/gitea_mirror_lib.sh"
# shellcheck source=gitea_submodule_lib.sh
. "$SCRIPT_DIR/gitea_submodule_lib.sh"
GITEA_OWNER="${GITEA_OWNER:-admin}"
GITEA_REPO="${GITEA_REPO:-android_cast}"
GITEA_MIRROR_URL="${GITEA_MIRROR_URL:-git://10.7.0.10/android_cast}"
GITEA_MAIN_MIRROR_INTERVAL="${GITEA_MAIN_MIRROR_INTERVAL:-1m}"
GITEA_SUBMODULE_MIRROR_INTERVAL="${GITEA_SUBMODULE_MIRROR_INTERVAL:-24h}"
GITEA_PRIVATE="${GITEA_PRIVATE:-false}"
GITEA_REF="${GITEA_REF:-HEAD}"
GITEA_SKIP_MAIN="${GITEA_SKIP_MAIN:-0}"
GITEA_SKIP_SUBMODULES="${GITEA_SKIP_SUBMODULES:-0}"
WORK_DIR="${WORK_DIR:-/tmp/gitea-migrate-$$}"
log() { printf '[gitea-migrate] %s\n' "$*"; }
warn() { printf '[gitea-migrate] WARN: %s\n' "$*" >&2; }
die() { printf '[gitea-migrate] ERROR: %s\n' "$*" >&2; exit 1; }
resolve_repo_dir() {
_arg="${1:-}"
if [ -n "$_arg" ]; then
[ -d "$_arg" ] || die "not a directory: $_arg"
printf '%s' "$(cd "$_arg" && pwd)"
return 0
fi
if [ -d "./.git" ] || [ -f "./.git" ]; then
pwd
return 0
fi
die "pass android_cast checkout path (needs .gitmodules and git history)"
}
migrate_main_repo() {
log "main mirror: $GITEA_OWNER/$GITEA_REPO <- $GITEA_MIRROR_URL (interval $GITEA_MAIN_MIRROR_INTERVAL)"
if git ls-remote "$GITEA_MIRROR_URL" HEAD >/dev/null 2>&1; then
log " upstream reachable"
else
warn " upstream not reachable from BE — check FE git daemon / [migrations] ALLOW_LOCALNETWORKS"
fi
if gitea_mirror_pull_repo "$GITEA_MIRROR_URL" "$GITEA_REPO" "$GITEA_OWNER" \
"$GITEA_MAIN_MIRROR_INTERVAL" "$WORK_DIR"; then
gitea_mirror_set_interval_api "$GITEA_OWNER" "$GITEA_REPO" "$GITEA_MAIN_MIRROR_INTERVAL" \
2>/dev/null || true
gitea_sync_mirror_api "$GITEA_OWNER" "$GITEA_REPO" || true
log " OK -> $(gitea_public_repo_url_for "$GITEA_OWNER" "$GITEA_REPO")"
return 0
fi
warn " main mirror FAILED"
return 1
}
migrate_submodule() {
_line="$1"
_repo="${_line%%|*}"
_rest="${_line#*|}"
_upstream="${_rest%%|*}"
_rest2="${_rest#*|}"
_path="${_rest2%%|*}"
_rest3="${_rest2#*|}"
_sha="${_rest3%%|*}"
_branch="${_rest3#*|}"
case "$_branch" in
"$_sha") _branch="" ;;
esac
log "submodule $_path ($GITEA_OWNER/$_repo)"
log " upstream: $_upstream"
log " pinned @ $GITEA_REF: $_sha"
if git ls-remote "$_upstream" "$_sha" >/dev/null 2>&1; then
log " commit on upstream"
else
warn " commit $_sha not on upstream (mirror may still fetch via branch tip)"
fi
if ! gitea_mirror_pull_repo "$_upstream" "$_repo" "$GITEA_OWNER" \
"$GITEA_SUBMODULE_MIRROR_INTERVAL" "$WORK_DIR"; then
warn " mirror create/sync FAILED for $_repo"
return 1
fi
gitea_mirror_set_interval_api "$GITEA_OWNER" "$_repo" "$GITEA_SUBMODULE_MIRROR_INTERVAL" \
2>/dev/null || true
gitea_sync_mirror_api "$GITEA_OWNER" "$_repo" || true
if gitea_mirror_has_commit "$GITEA_OWNER" "$_repo" "$_sha"; then
log " OK pinned commit present -> $(gitea_public_repo_url_for "$GITEA_OWNER" "$_repo")"
return 0
fi
warn " mirror ok but pinned $_sha not visible yet — retry: $SCRIPT_DIR/gitea_sync_mirror.sh $_repo"
return 2
}
print_fast_sync_guide() {
_sync_sh="$SCRIPT_DIR/gitea_sync_mirror.sh"
_hook="$SCRIPT_DIR/fe-android_cast-post-receive.example"
cat <<EOF
Fast sync for local main mirror ($GITEA_REPO only)
-------------------------------------------------
Gitea default minimum mirror interval is 10m. For 1m polling set in live app.ini:
[mirror]
MIN_INTERVAL = 1m
[cron.update_mirrors]
SCHEDULE = @every 1m
RUN_AT_START = true
sudo rc-service gitea restart
Per-repo interval for $GITEA_REPO is already set to $GITEA_MAIN_MIRROR_INTERVAL by this script.
Immediate sync (push-triggered or cron every minute on BE):
sudo $_sync_sh $GITEA_REPO
Sync all mirrors:
sudo $_sync_sh --all
FE git post-receive hook (git on 10.7.0.10 → trigger BE sync):
See $_hook
Copy to FE bare repo hooks/post-receive and set GITEA_MIRROR_SYNC_CMD (ssh or curl).
Submodule mirrors use interval $GITEA_SUBMODULE_MIRROR_INTERVAL; pinned SHAs live in
android_cast gitlinks — mirrors hold full upstream history so those commits are fetchable.
Local clone rewrite (optional, .gitmodules unchanged):
$SCRIPT_DIR/gitea_submodule_local_rewrite.sh --apply
EOF
}
# --- main ---
_repo_arg=""
while [ $# -gt 0 ]; do
case "$1" in
-h|--help)
sed -n '2,18p' "$0"
exit 0
;;
--skip-main) GITEA_SKIP_MAIN=1; shift ;;
--skip-submodules) GITEA_SKIP_SUBMODULES=1; shift ;;
--ref)
shift
GITEA_REF="${1:-HEAD}"
shift
;;
-*)
die "unknown option: $1"
;;
*)
_repo_arg="$1"
shift
;;
esac
done
gitea_load_config || exit 1
gitea_ensure_cli_access || exit 1
if ! gitea_user_exists "$GITEA_OWNER"; then
_fb="$(gitea_pick_owner_username)"
[ -n "$_fb" ] || die "no Gitea user; set GITEA_OWNER=admin"
warn "GITEA_OWNER=$GITEA_OWNER not found; using $_fb"
GITEA_OWNER="$_fb"
fi
if command -v rc-service >/dev/null 2>&1; then
rc-service gitea status 2>/dev/null | grep -qi started || rc-service gitea start 2>/dev/null || true
fi
mkdir -p "$WORK_DIR"
trap 'rm -rf "$WORK_DIR"' EXIT INT HUP TERM
log "ini=$GITEA_APP_INI owner=$GITEA_OWNER ref=$GITEA_REF"
_main_ok=0
_main_fail=0
_sub_ok=0
_sub_warn=0
_sub_fail=0
if [ "$GITEA_SKIP_MAIN" -eq 0 ]; then
if migrate_main_repo; then
_main_ok=1
else
_main_fail=1
fi
else
log "skipping main mirror (--skip-main)"
fi
if [ "$GITEA_SKIP_SUBMODULES" -eq 0 ]; then
_repo_dir="$(resolve_repo_dir "$_repo_arg")"
log "discovering external submodules in $_repo_dir"
_sub_list="$WORK_DIR/submodules.list"
: > "$_sub_list"
gitea_submodule_discover_external "$_repo_dir" "$GITEA_REF" > "$_sub_list" || true
if [ ! -s "$_sub_list" ]; then
warn "no external submodules at $GITEA_REF (empty .gitmodules or only local URLs)"
else
while IFS= read -r _line; do
[ -n "$_line" ] || continue
gitea_submodule_entry_valid "$_line" || continue
_rc=0
migrate_submodule "$_line" || _rc=$?
case "$_rc" in
0) _sub_ok=$((_sub_ok + 1)) ;;
2) _sub_warn=$((_sub_warn + 1)) ;;
*) _sub_fail=$((_sub_fail + 1)) ;;
esac
done < "$_sub_list"
fi
else
log "skipping submodule mirrors (--skip-submodules)"
fi
log "summary: main ok=$_main_ok fail=$_main_fail | submodules ok=$_sub_ok warn=$_sub_warn fail=$_sub_fail"
print_fast_sync_guide
[ "$_main_fail" -eq 0 ] && [ "$_sub_fail" -eq 0 ]

View File

@@ -149,6 +149,36 @@ gitea_sync_mirror_api() {
-H "Authorization: token $_token" | grep -q '^20' -H "Authorization: token $_token" | grep -q '^20'
} }
gitea_mirror_set_interval_api() {
_owner="$1"
_repo="$2"
_interval="$3"
command -v curl >/dev/null 2>&1 || return 1
_token="${GITEA_ADMIN_TOKEN:-}"
[ -n "$_token" ] || _token="$(gitea_resolve_push_token)" || return 1
_api="$(gitea_api_base)"
_payload="{\"mirror_interval\":\"$_interval\"}"
_http="$(curl -sS -o /tmp/gitea-mirror-interval-$$.json -w '%{http_code}' \
-X PATCH "$_api/repos/$_owner/$_repo" \
-H "Authorization: token $_token" \
-H "Content-Type: application/json" \
-d "$_payload")"
case "$_http" in
200|204) return 0 ;;
*) return 1 ;;
esac
}
# True if commit SHA is reachable on the mirror (after sync).
gitea_mirror_has_commit() {
_owner="$1"
_repo="$2"
_sha="$3"
command -v git >/dev/null 2>&1 || return 1
_url="$(gitea_local_git_url_for "$_owner" "$_repo")"
git ls-remote "$_url" "$_sha" 2>/dev/null | grep -q "$_sha"
}
# Mirror one upstream into Gitea (API migrate, else git push). Returns 0 on success. # Mirror one upstream into Gitea (API migrate, else git push). Returns 0 on success.
gitea_mirror_pull_repo() { gitea_mirror_pull_repo() {
_clone_url="$1" _clone_url="$1"
@@ -162,6 +192,7 @@ gitea_mirror_pull_repo() {
&& return 0 && return 0
_api_rc=$? _api_rc=$?
if [ "$_api_rc" -eq 2 ]; then if [ "$_api_rc" -eq 2 ]; then
gitea_mirror_set_interval_api "$_owner" "$_repo" "$_interval" 2>/dev/null || true
gitea_sync_mirror_api "$_owner" "$_repo" && return 0 gitea_sync_mirror_api "$_owner" "$_repo" && return 0
return 0 return 0
fi fi

View File

@@ -0,0 +1,116 @@
#!/bin/ash
# Discover git submodules (upstream URL + pinned SHA) from a checkout.
# shellcheck shell=sh
# Space-separated host/prefix patterns treated as "ours" (skipped for external mirrors).
GITEA_LOCAL_URL_PATTERNS="${GITEA_LOCAL_URL_PATTERNS:-10.7.0.10 10.7.16. apps.f0xx.org f0xx.org 127.0.0.1 localhost}"
gitea_submodule_is_local_url() {
_url="$1"
case "$_url" in
/*|./*|../*) return 0 ;;
esac
for _pat in $GITEA_LOCAL_URL_PATTERNS; do
case "$_url" in
*"$_pat"*) return 0 ;;
esac
done
return 1
}
gitea_submodule_is_external_url() {
gitea_submodule_is_local_url "$1" && return 1
return 0
}
# .gitmodules -> repo|url|path|branch (repo = basename of path). stdout only.
gitea_submodule_parse_gitmodules() {
_file="$1"
[ -r "$_file" ] || return 1
awk '
function trim(s) {
sub(/^[ \t]+/, "", s); sub(/[ \t]+$/, "", s)
return s
}
/^[ \t]*path[ \t]*=/ {
p = $0; sub(/^[^=]*=/, "", p); path = trim(p)
n = path; sub(".*/", "", n)
next
}
/^[ \t]*url[ \t]*=/ {
u = $0; sub(/^[^=]*=/, "", u); url = trim(u)
next
}
/^[ \t]*branch[ \t]*=/ {
b = $0; sub(/^[^=]*=/, "", b); branch = trim(b)
next
}
/^\[/ {
if (path != "" && url != "" && n != "") {
if (branch != "") print n "|" url "|" path "|" branch
else print n "|" url "|" path
}
path = ""; url = ""; branch = ""; n = ""
}
END {
if (path != "" && url != "" && n != "") {
if (branch != "") print n "|" url "|" path "|" branch
else print n "|" url "|" path
}
}
' "$_file"
}
# repo|url|path|sha[|branch] for each external submodule with a gitlink at REF.
gitea_submodule_discover_external() {
_repo_dir="$1"
_ref="${2:-HEAD}"
_gitmodules="$_repo_dir/.gitmodules"
command -v git >/dev/null 2>&1 || return 1
git -C "$_repo_dir" rev-parse --verify "$_ref" >/dev/null 2>&1 || return 1
[ -r "$_gitmodules" ] || return 1
gitea_submodule_parse_gitmodules "$_gitmodules" | while IFS= read -r _line; do
[ -n "$_line" ] || continue
_repo="${_line%%|*}"
_rest="${_line#*|}"
_url="${_rest%%|*}"
_rest2="${_rest#*|}"
_path="${_rest2%%|*}"
_branch="${_rest2#*|}"
case "$_branch" in
"$_path") _branch="" ;;
esac
gitea_submodule_is_external_url "$_url" || continue
_sha="$(git -C "$_repo_dir" rev-parse "$_ref:$_path" 2>/dev/null)" || continue
case "$_sha" in
??000000000000000000000000000000000000) continue ;;
esac
if [ -n "$_branch" ]; then
printf '%s|%s|%s|%s|%s\n' "$_repo" "$_url" "$_path" "$_sha" "$_branch"
else
printf '%s|%s|%s|%s\n' "$_repo" "$_url" "$_path" "$_sha"
fi
done
}
gitea_submodule_entry_valid() {
case "$1" in
*'|'*) ;;
*) return 1 ;;
esac
_repo="${1%%|*}"
_rest="${1#*|}"
_url="${_rest%%|*}"
_rest2="${_rest#*|}"
_path="${_rest2%%|*}"
[ -n "$_repo" ] && [ -n "$_url" ] && [ -n "$_path" ] || return 1
case "$_repo" in
*' '*|*'['*|*']'*) return 1 ;;
esac
return 0
}

View File

@@ -0,0 +1,63 @@
#!/bin/ash
#
# Trigger Gitea pull-mirror sync (one repo or all). For BE cron / FE post-receive hook.
#
# Usage:
# sudo ./gitea_sync_mirror.sh android_cast
# sudo ./gitea_sync_mirror.sh android_cast libvpx opus
# sudo ./gitea_sync_mirror.sh --all
#
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
# shellcheck source=gitea_ini.sh
. "$SCRIPT_DIR/gitea_ini.sh"
# shellcheck source=gitea_mirror_lib.sh
. "$SCRIPT_DIR/gitea_mirror_lib.sh"
GITEA_OWNER="${GITEA_OWNER:-admin}"
log() { printf '[gitea-sync] %s\n' "$*"; }
warn() { printf '[gitea-sync] WARN: %s\n' "$*" >&2; }
die() { printf '[gitea-sync] ERROR: %s\n' "$*" >&2; exit 1; }
gitea_load_config || exit 1
gitea_ensure_cli_access || exit 1
_sync_all=0
_repos=""
while [ $# -gt 0 ]; do
case "$1" in
-h|--help)
sed -n '2,10p' "$0"
exit 0
;;
--all) _sync_all=1; shift ;;
-*) die "unknown option: $1" ;;
*) _repos="$_repos $1"; shift ;;
esac
done
if [ "$_sync_all" -eq 1 ]; then
log "sync all mirrors via gitea admin repo-sync-mirrors"
gitea_cli "admin" "repo-sync-mirrors"
exit $?
fi
if [ -z "$(echo "$_repos" | tr -d ' ')" ]; then
_repos="android_cast"
fi
_fail=0
for _repo in $_repos; do
log "mirror-sync $GITEA_OWNER/$_repo"
if gitea_sync_mirror_api "$GITEA_OWNER" "$_repo"; then
log " OK"
else
warn " FAILED"
_fail=1
fi
done
exit "$_fail"