1
0
mirror of git://f0xx.org/android_cast synced 2026-07-29 06:39:09 +03:00

gitea struggle

This commit is contained in:
Anton Afanasyeu
2026-06-10 22:15:17 +02:00
parent 89a0365038
commit 91cac577e8
5 changed files with 327 additions and 35 deletions

View File

@@ -178,6 +178,24 @@ sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh .
GITEA_REF=next sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh . GITEA_REF=next sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh .
``` ```
### Troubleshooting: only `android_cast` visible (Repositories 1)
`gitea_attach_remote.sh` mirrors **only the main repo**. Submodule repos need a separate step.
```bash
# 1) See what BE would mirror / what exists in Gitea:
sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_diagnose_mirrors.sh .
# 2) Create submodule mirrors (main already exists):
sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh . --skip-main
# or:
sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_mirror_submodules.sh .gitmodules
```
Expect **4 repos** under `admin`: `android_cast`, `libvpx`, `opus`, `speex`. If migrate summary shows `submodules ok=0` and a WARN about empty discovery, `.gitmodules` is missing on the BE checkout — the updated migrate script falls back to `submodules.defaults.conf`.
In Gitea UI, use **Explore** or **Site Administration → Repositories** if the profile counter hides mirror repos.
### Fast sync for local `android_cast` mirror only ### Fast sync for local `android_cast` mirror only
Gitea **default minimum** pull interval is **10m**. To use **1m** for the FE git mirror: Gitea **default minimum** pull interval is **10m**. To use **1m** for the FE git mirror:

View File

@@ -0,0 +1,105 @@
#!/bin/ash
#
# Debug why submodule mirrors are missing in Gitea UI.
#
# Usage:
# sudo ./gitea_diagnose_mirrors.sh
# sudo ./gitea_diagnose_mirrors.sh /var/www/.../android_cast
# sudo ./gitea_diagnose_mirrors.sh /var/www/.../android_cast/.gitmodules
#
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
# shellcheck source=gitea_ini.sh
. "$SCRIPT_DIR/gitea_ini.sh"
# shellcheck source=gitea_mirror_lib.sh
. "$SCRIPT_DIR/gitea_mirror_lib.sh"
# shellcheck source=gitea_submodule_lib.sh
. "$SCRIPT_DIR/gitea_submodule_lib.sh"
GITEA_OWNER="${GITEA_OWNER:-admin}"
GITEA_REF="${GITEA_REF:-HEAD}"
GITEA_SUBMODULES_CONF="${GITEA_SUBMODULES_CONF:-$SCRIPT_DIR/submodules.defaults.conf}"
log() { printf '[gitea-diagnose] %s\n' "$*"; }
warn() { printf '[gitea-diagnose] WARN: %s\n' "$*" >&2; }
_arg="${1:-.}"
_repo_dir=""
_gitmodules=""
case "$_arg" in
*.gitmodules)
_gitmodules="$(cd "$(dirname "$_arg")" && pwd)/$(basename "$_arg")"
_parent="$(dirname "$_gitmodules")"
[ -d "$_parent/.git" ] || [ -f "$_parent/.git" ] && _repo_dir="$(cd "$_parent" && pwd)" || _repo_dir="$_parent"
;;
*)
[ -d "$_arg" ] || { warn "not a directory: $_arg"; exit 1; }
_repo_dir="$(cd "$_arg" && pwd)"
_gitmodules="$_repo_dir/.gitmodules"
;;
esac
gitea_load_config || exit 1
gitea_ensure_cli_access || exit 1
log "ini=$GITEA_APP_INI owner=$GITEA_OWNER"
log "checkout=$_repo_dir"
log "gitmodules=$_gitmodules"
if [ -r "$_gitmodules" ]; then
log ".gitmodules present ($(wc -l < "$_gitmodules" | tr -d ' ') lines)"
else
warn ".gitmodules MISSING on BE — submodule migrate cannot discover entries"
fi
if command -v git >/dev/null 2>&1 && [ -d "$_repo_dir/.git" ] || [ -f "$_repo_dir/.git" ]; then
log "git branch: $(git -C "$_repo_dir" branch --show-current 2>/dev/null || echo '?')"
log "git HEAD: $(git -C "$_repo_dir" rev-parse HEAD 2>/dev/null || echo '?')"
for _p in third-party/libvpx third-party/opus third-party/speex; do
_sha="$(git -C "$_repo_dir" rev-parse "$GITEA_REF:$_p" 2>/dev/null)" || _sha="(no gitlink)"
log " $GITEA_REF:$_p -> $_sha"
done
else
warn "not a git checkout — only .gitmodules / defaults.conf can be used"
fi
log "collect_entries (what migrate would mirror):"
gitea_submodule_collect_entries "$_repo_dir" "$GITEA_REF" "$_gitmodules" "$GITEA_SUBMODULES_CONF" \
| while IFS= read -r _e; do
[ -n "$_e" ] && log " $_e"
done
log "Gitea repos via API:"
if _names="$(gitea_list_repos_api "$GITEA_OWNER" 2>/dev/null)"; then
echo "$_names" | while read -r _r; do
[ -n "$_r" ] && log " $GITEA_OWNER/$_r"
done
log "count=$(echo "$_names" | grep -c . 2>/dev/null || echo 0)"
else
warn "API list failed"
fi
log "on-disk bare repos:"
if [ -d "${GITEA_REPO_ROOT:-/var/lib/gitea/data/gitea-repositories}/$GITEA_OWNER" ]; then
ls -1 "${GITEA_REPO_ROOT}/$GITEA_OWNER" 2>/dev/null | while read -r _d; do
log " $GITEA_OWNER/$_d"
done
else
warn "no directory ${GITEA_REPO_ROOT}/$GITEA_OWNER"
fi
cat <<EOF
If count=1 (only android_cast):
• gitea_attach_remote.sh only mirrors the main repo — run submodule step:
sudo ./gitea_migrate_project.sh . --skip-main
or:
sudo ./gitea_mirror_submodules.sh .gitmodules
• If collect_entries is empty: sync .gitmodules to BE or use submodules.defaults.conf
Gitea UI: profile "Repositories N" may exclude mirrors on some versions —
try Explore → search libvpx, or Site Administration → Repositories.
EOF

View File

@@ -32,24 +32,46 @@ GITEA_PRIVATE="${GITEA_PRIVATE:-false}"
GITEA_REF="${GITEA_REF:-HEAD}" GITEA_REF="${GITEA_REF:-HEAD}"
GITEA_SKIP_MAIN="${GITEA_SKIP_MAIN:-0}" GITEA_SKIP_MAIN="${GITEA_SKIP_MAIN:-0}"
GITEA_SKIP_SUBMODULES="${GITEA_SKIP_SUBMODULES:-0}" GITEA_SKIP_SUBMODULES="${GITEA_SKIP_SUBMODULES:-0}"
GITEA_SUBMODULES_CONF="${GITEA_SUBMODULES_CONF:-$SCRIPT_DIR/submodules.defaults.conf}"
WORK_DIR="${WORK_DIR:-/tmp/gitea-migrate-$$}" WORK_DIR="${WORK_DIR:-/tmp/gitea-migrate-$$}"
log() { printf '[gitea-migrate] %s\n' "$*"; } log() { printf '[gitea-migrate] %s\n' "$*"; }
warn() { printf '[gitea-migrate] WARN: %s\n' "$*" >&2; } warn() { printf '[gitea-migrate] WARN: %s\n' "$*" >&2; }
die() { printf '[gitea-migrate] ERROR: %s\n' "$*" >&2; exit 1; } die() { printf '[gitea-migrate] ERROR: %s\n' "$*" >&2; exit 1; }
resolve_repo_dir() { # Sets GITEA_REPO_DIR and GITEA_GITMODULES_FILE.
resolve_repo_inputs() {
_arg="${1:-}" _arg="${1:-}"
GITEA_REPO_DIR=""
GITEA_GITMODULES_FILE=""
if [ -n "$_arg" ]; then if [ -n "$_arg" ]; then
[ -d "$_arg" ] || die "not a directory: $_arg" case "$_arg" in
printf '%s' "$(cd "$_arg" && pwd)" *.gitmodules)
[ -r "$_arg" ] || die "cannot read $_arg"
GITEA_GITMODULES_FILE="$(cd "$(dirname "$_arg")" && pwd)/$(basename "$_arg")"
_parent="$(dirname "$GITEA_GITMODULES_FILE")"
if [ -d "$_parent/.git" ] || [ -f "$_parent/.git" ]; then
GITEA_REPO_DIR="$(cd "$_parent" && pwd)"
else
GITEA_REPO_DIR="$_parent"
fi
return 0 return 0
;;
*)
[ -d "$_arg" ] || die "not a directory: $_arg"
GITEA_REPO_DIR="$(cd "$_arg" && pwd)"
GITEA_GITMODULES_FILE="$GITEA_REPO_DIR/.gitmodules"
return 0
;;
esac
fi fi
if [ -d "./.git" ] || [ -f "./.git" ]; then if [ -d "./.git" ] || [ -f "./.git" ]; then
pwd GITEA_REPO_DIR="$(pwd)"
GITEA_GITMODULES_FILE="$GITEA_REPO_DIR/.gitmodules"
return 0 return 0
fi fi
die "pass android_cast checkout path (needs .gitmodules and git history)" die "pass android_cast checkout path or .gitmodules (needs .gitmodules)"
} }
migrate_main_repo() { migrate_main_repo() {
@@ -88,13 +110,16 @@ migrate_submodule() {
log "submodule $_path ($GITEA_OWNER/$_repo)" log "submodule $_path ($GITEA_OWNER/$_repo)"
log " upstream: $_upstream" log " upstream: $_upstream"
if [ "$_sha" = "-" ]; then
log " pinned @ $GITEA_REF: (unknown — no gitlink in checkout; mirroring upstream anyway)"
else
log " pinned @ $GITEA_REF: $_sha" log " pinned @ $GITEA_REF: $_sha"
if git ls-remote "$_upstream" "$_sha" >/dev/null 2>&1; then if git ls-remote "$_upstream" "$_sha" >/dev/null 2>&1; then
log " commit on upstream" log " commit on upstream"
else else
warn " commit $_sha not on upstream (mirror may still fetch via branch tip)" warn " commit $_sha not on upstream (mirror may still fetch via branch tip)"
fi fi
fi
if ! gitea_mirror_pull_repo "$_upstream" "$_repo" "$GITEA_OWNER" \ if ! gitea_mirror_pull_repo "$_upstream" "$_repo" "$GITEA_OWNER" \
"$GITEA_SUBMODULE_MIRROR_INTERVAL" "$WORK_DIR"; then "$GITEA_SUBMODULE_MIRROR_INTERVAL" "$WORK_DIR"; then
@@ -106,6 +131,11 @@ migrate_submodule() {
2>/dev/null || true 2>/dev/null || true
gitea_sync_mirror_api "$GITEA_OWNER" "$_repo" || true gitea_sync_mirror_api "$GITEA_OWNER" "$_repo" || true
if [ "$_sha" = "-" ]; then
log " OK mirror -> $(gitea_public_repo_url_for "$GITEA_OWNER" "$_repo")"
return 0
fi
if gitea_mirror_has_commit "$GITEA_OWNER" "$_repo" "$_sha"; then if gitea_mirror_has_commit "$GITEA_OWNER" "$_repo" "$_sha"; then
log " OK pinned commit present -> $(gitea_public_repo_url_for "$GITEA_OWNER" "$_repo")" log " OK pinned commit present -> $(gitea_public_repo_url_for "$GITEA_OWNER" "$_repo")"
return 0 return 0
@@ -115,6 +145,23 @@ migrate_submodule() {
return 2 return 2
} }
print_gitea_repo_inventory() {
log "Gitea repos for $GITEA_OWNER (API):"
_names="$(gitea_list_repos_api "$GITEA_OWNER" 2>/dev/null)" || _names=""
if [ -z "$_names" ]; then
warn " could not list repos via API (token/CLI)"
return 1
fi
_n=0
echo "$_names" | while read -r _r; do
[ -n "$_r" ] || continue
log " - $GITEA_OWNER/$_r"
_n=$((_n + 1))
done
_count="$(echo "$_names" | grep -c . 2>/dev/null || echo 0)"
log "total listed: $_count (expect android_cast + libvpx + opus + speex)"
}
print_fast_sync_guide() { print_fast_sync_guide() {
_sync_sh="$SCRIPT_DIR/gitea_sync_mirror.sh" _sync_sh="$SCRIPT_DIR/gitea_sync_mirror.sh"
_hook="$SCRIPT_DIR/fe-android_cast-post-receive.example" _hook="$SCRIPT_DIR/fe-android_cast-post-receive.example"
@@ -216,14 +263,19 @@ else
fi fi
if [ "$GITEA_SKIP_SUBMODULES" -eq 0 ]; then if [ "$GITEA_SKIP_SUBMODULES" -eq 0 ]; then
_repo_dir="$(resolve_repo_dir "$_repo_arg")" resolve_repo_inputs "$_repo_arg"
log "discovering external submodules in $_repo_dir" log "submodule inputs: repo=$GITEA_REPO_DIR gitmodules=$GITEA_GITMODULES_FILE ref=$GITEA_REF"
_sub_list="$WORK_DIR/submodules.list" _sub_list="$WORK_DIR/submodules.list"
: > "$_sub_list" gitea_submodule_collect_entries "$GITEA_REPO_DIR" "$GITEA_REF" \
gitea_submodule_discover_external "$_repo_dir" "$GITEA_REF" > "$_sub_list" || true "$GITEA_GITMODULES_FILE" "$GITEA_SUBMODULES_CONF" > "$_sub_list" || true
if [ ! -s "$_sub_list" ]; then if [ ! -s "$_sub_list" ]; then
warn "no external submodules at $GITEA_REF (empty .gitmodules or only local URLs)" die "no external submodules found — check .gitmodules on BE or $GITEA_SUBMODULES_CONF"
else fi
log "submodules to mirror:"
while IFS= read -r _preview; do
[ -n "$_preview" ] || continue
log " $_preview"
done < "$_sub_list"
while IFS= read -r _line; do while IFS= read -r _line; do
[ -n "$_line" ] || continue [ -n "$_line" ] || continue
gitea_submodule_entry_valid "$_line" || continue gitea_submodule_entry_valid "$_line" || continue
@@ -235,12 +287,14 @@ if [ "$GITEA_SKIP_SUBMODULES" -eq 0 ]; then
*) _sub_fail=$((_sub_fail + 1)) ;; *) _sub_fail=$((_sub_fail + 1)) ;;
esac esac
done < "$_sub_list" done < "$_sub_list"
fi
else else
log "skipping submodule mirrors (--skip-submodules)" log "skipping submodule mirrors (--skip-submodules)"
fi fi
log "summary: main ok=$_main_ok fail=$_main_fail | submodules ok=$_sub_ok warn=$_sub_warn fail=$_sub_fail" log "summary: main ok=$_main_ok fail=$_main_fail | submodules ok=$_sub_ok warn=$_sub_warn fail=$_sub_fail"
print_gitea_repo_inventory
print_fast_sync_guide print_fast_sync_guide
[ "$_main_fail" -eq 0 ] && [ "$_sub_fail" -eq 0 ] _sub_migrated=$((_sub_ok + _sub_warn))
[ "$_main_fail" -eq 0 ] && [ "$_sub_fail" -eq 0 ] \
&& { [ "$GITEA_SKIP_SUBMODULES" -eq 1 ] || [ "$_sub_migrated" -gt 0 ]; }

View File

@@ -115,6 +115,11 @@ EOF
if [ "$_http" = "409" ] || [ "$_http" = "422" ]; then if [ "$_http" = "409" ] || [ "$_http" = "422" ]; then
return 2 return 2
fi fi
if [ -r "/tmp/gitea-migrate-$$.json" ]; then
printf '[gitea-mirror] migrate HTTP %s: ' "$_http" >&2
head -c 400 "/tmp/gitea-migrate-$$.json" >&2 || true
printf '\n' >&2
fi
return 1 return 1
} }
@@ -137,6 +142,30 @@ gitea_mirror_push_git() {
return 0 return 0
} }
gitea_repo_exists_api() {
_owner="$1"
_repo="$2"
command -v curl >/dev/null 2>&1 || return 1
_token="${GITEA_ADMIN_TOKEN:-}"
[ -n "$_token" ] || _token="$(gitea_resolve_push_token)" || return 1
_api="$(gitea_api_base)"
_http="$(curl -sS -o /dev/null -w '%{http_code}' \
"$_api/repos/$_owner/$_repo" \
-H "Authorization: token $_token")"
[ "$_http" = "200" ]
}
gitea_list_repos_api() {
_owner="$1"
command -v curl >/dev/null 2>&1 || return 1
_token="${GITEA_ADMIN_TOKEN:-}"
[ -n "$_token" ] || _token="$(gitea_resolve_push_token)" || return 1
_api="$(gitea_api_base)"
curl -sS "$_api/users/$_owner/repos?limit=50" \
-H "Authorization: token $_token" \
| awk -F'"' '/"name":/ { print $4 }'
}
gitea_sync_mirror_api() { gitea_sync_mirror_api() {
_owner="$1" _owner="$1"
_repo="$2" _repo="$2"
@@ -192,10 +221,14 @@ gitea_mirror_pull_repo() {
&& return 0 && return 0
_api_rc=$? _api_rc=$?
if [ "$_api_rc" -eq 2 ]; then if [ "$_api_rc" -eq 2 ]; then
if gitea_repo_exists_api "$_owner" "$_repo"; then
gitea_mirror_set_interval_api "$_owner" "$_repo" "$_interval" 2>/dev/null || true gitea_mirror_set_interval_api "$_owner" "$_repo" "$_interval" 2>/dev/null || true
gitea_sync_mirror_api "$_owner" "$_repo" && return 0 gitea_sync_mirror_api "$_owner" "$_repo" && return 0
return 0 return 0
fi fi
printf '[gitea-mirror] migrate rejected for %s/%s but repo missing — trying git push\n' \
"$_owner" "$_repo" >&2
fi
mkdir -p "$_work_dir" mkdir -p "$_work_dir"
gitea_mirror_push_git "$_owner" "$_repo" "$_clone_url" "$_work_dir" || _rc=1 gitea_mirror_push_git "$_owner" "$_repo" "$_clone_url" "$_work_dir" || _rc=1
return "$_rc" return "$_rc"

View File

@@ -61,14 +61,11 @@ gitea_submodule_parse_gitmodules() {
' "$_file" ' "$_file"
} }
# repo|url|path|sha[|branch] for each external submodule with a gitlink at REF. # repo|url|path|sha[|branch] external entries from .gitmodules (sha may be "-" if unknown).
gitea_submodule_discover_external() { gitea_submodule_discover_from_gitmodules() {
_repo_dir="$1" _gitmodules="$1"
_ref="${2:-HEAD}" _repo_dir="${2:-}"
_gitmodules="$_repo_dir/.gitmodules" _ref="${3:-HEAD}"
command -v git >/dev/null 2>&1 || return 1
git -C "$_repo_dir" rev-parse --verify "$_ref" >/dev/null 2>&1 || return 1
[ -r "$_gitmodules" ] || return 1 [ -r "$_gitmodules" ] || return 1
gitea_submodule_parse_gitmodules "$_gitmodules" | while IFS= read -r _line; do gitea_submodule_parse_gitmodules "$_gitmodules" | while IFS= read -r _line; do
@@ -85,10 +82,15 @@ gitea_submodule_discover_external() {
gitea_submodule_is_external_url "$_url" || continue gitea_submodule_is_external_url "$_url" || continue
_sha="$(git -C "$_repo_dir" rev-parse "$_ref:$_path" 2>/dev/null)" || continue _sha="-"
case "$_sha" in if [ -n "$_repo_dir" ] && command -v git >/dev/null 2>&1 \
??000000000000000000000000000000000000) continue ;; && git -C "$_repo_dir" rev-parse --verify "$_ref" >/dev/null 2>&1; then
_got="$(git -C "$_repo_dir" rev-parse "$_ref:$_path" 2>/dev/null)" || _got=""
case "$_got" in
??000000000000000000000000000000000000) _sha="-" ;;
????????????????????????????????????????*) _sha="$_got" ;;
esac esac
fi
if [ -n "$_branch" ]; then if [ -n "$_branch" ]; then
printf '%s|%s|%s|%s|%s\n' "$_repo" "$_url" "$_path" "$_sha" "$_branch" printf '%s|%s|%s|%s|%s\n' "$_repo" "$_url" "$_path" "$_sha" "$_branch"
@@ -98,6 +100,86 @@ gitea_submodule_discover_external() {
done done
} }
# repo|url|path|sha[|branch] for each external submodule with a gitlink at REF.
gitea_submodule_discover_external() {
_repo_dir="$1"
_ref="${2:-HEAD}"
_gitmodules="$_repo_dir/.gitmodules"
command -v git >/dev/null 2>&1 || return 1
git -C "$_repo_dir" rev-parse --verify "$_ref" >/dev/null 2>&1 || return 1
[ -r "$_gitmodules" ] || return 1
gitea_submodule_discover_from_gitmodules "$_gitmodules" "$_repo_dir" "$_ref" \
| while IFS= read -r _line; do
_sha="${_line##*|}"
case "$_sha" in
-) continue ;;
esac
printf '%s\n' "$_line"
done
}
# repo|url|path|sha[|branch] from submodules.defaults.conf (sha resolved when possible).
gitea_submodule_load_defaults() {
_conf="$1"
_repo_dir="${2:-}"
_ref="${3:-HEAD}"
[ -r "$_conf" ] || return 1
grep -v '^[[:space:]]*#' "$_conf" | grep -v '^[[:space:]]*$' | while IFS= read -r _line; do
[ -n "$_line" ] || continue
_repo="${_line%%|*}"
_rest="${_line#*|}"
_url="${_rest%%|*}"
_rest2="${_rest#*|}"
_path="${_rest2%%|*}"
_branch="${_rest2#*|}"
case "$_branch" in
"$_path") _branch="" ;;
esac
gitea_submodule_is_external_url "$_url" || continue
_sha="-"
if [ -n "$_repo_dir" ] && command -v git >/dev/null 2>&1 \
&& git -C "$_repo_dir" rev-parse --verify "$_ref" >/dev/null 2>&1; then
_got="$(git -C "$_repo_dir" rev-parse "$_ref:$_path" 2>/dev/null)" || _got=""
case "$_got" in
????????????????????????????????????????*) _sha="$_got" ;;
esac
fi
if [ -n "$_branch" ]; then
printf '%s|%s|%s|%s|%s\n' "$_repo" "$_url" "$_path" "$_sha" "$_branch"
else
printf '%s|%s|%s|%s\n' "$_repo" "$_url" "$_path" "$_sha"
fi
done
}
# Merge discovery sources; first win per repo name. stdout: repo|url|path|sha[|branch]
gitea_submodule_collect_entries() {
_repo_dir="${1:-}"
_ref="${2:-HEAD}"
_gitmodules="${3:-}"
_defaults="${4:-}"
_work="$(mktemp /tmp/gitea-submods-XXXXXX)"
if [ -n "$_gitmodules" ] && [ -r "$_gitmodules" ]; then
gitea_submodule_discover_from_gitmodules "$_gitmodules" "$_repo_dir" "$_ref" >> "$_work" 2>/dev/null || true
elif [ -n "$_repo_dir" ] && [ -r "$_repo_dir/.gitmodules" ]; then
gitea_submodule_discover_from_gitmodules "$_repo_dir/.gitmodules" "$_repo_dir" "$_ref" >> "$_work" 2>/dev/null || true
fi
if [ ! -s "$_work" ] && [ -n "$_defaults" ] && [ -r "$_defaults" ]; then
gitea_submodule_load_defaults "$_defaults" "$_repo_dir" "$_ref" >> "$_work" 2>/dev/null || true
elif [ -n "$_defaults" ] && [ -r "$_defaults" ]; then
gitea_submodule_load_defaults "$_defaults" "$_repo_dir" "$_ref" >> "$_work" 2>/dev/null || true
fi
awk -F'|' '!seen[$1]++' "$_work"
rm -f "$_work"
}
gitea_submodule_entry_valid() { gitea_submodule_entry_valid() {
case "$1" in case "$1" in
*'|'*) ;; *'|'*) ;;