diff --git a/examples/crash_reporter/backend/scripts/gitea/README.md b/examples/crash_reporter/backend/scripts/gitea/README.md index 000e913..93e02a6 100644 --- a/examples/crash_reporter/backend/scripts/gitea/README.md +++ b/examples/crash_reporter/backend/scripts/gitea/README.md @@ -178,6 +178,24 @@ sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh . GITEA_REF=next sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh . ``` +### Troubleshooting: only `android_cast` visible (Repositories 1) + +`gitea_attach_remote.sh` mirrors **only the main repo**. Submodule repos need a separate step. + +```bash +# 1) See what BE would mirror / what exists in Gitea: +sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_diagnose_mirrors.sh . + +# 2) Create submodule mirrors (main already exists): +sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh . --skip-main +# or: +sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_mirror_submodules.sh .gitmodules +``` + +Expect **4 repos** under `admin`: `android_cast`, `libvpx`, `opus`, `speex`. If migrate summary shows `submodules ok=0` and a WARN about empty discovery, `.gitmodules` is missing on the BE checkout — the updated migrate script falls back to `submodules.defaults.conf`. + +In Gitea UI, use **Explore** or **Site Administration → Repositories** if the profile counter hides mirror repos. + ### Fast sync for local `android_cast` mirror only Gitea **default minimum** pull interval is **10m**. To use **1m** for the FE git mirror: diff --git a/examples/crash_reporter/backend/scripts/gitea/gitea_diagnose_mirrors.sh b/examples/crash_reporter/backend/scripts/gitea/gitea_diagnose_mirrors.sh new file mode 100755 index 0000000..2737dad --- /dev/null +++ b/examples/crash_reporter/backend/scripts/gitea/gitea_diagnose_mirrors.sh @@ -0,0 +1,105 @@ +#!/bin/ash +# +# Debug why submodule mirrors are missing in Gitea UI. +# +# Usage: +# sudo ./gitea_diagnose_mirrors.sh +# sudo ./gitea_diagnose_mirrors.sh /var/www/.../android_cast +# sudo ./gitea_diagnose_mirrors.sh /var/www/.../android_cast/.gitmodules +# +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=gitea_ini.sh +. "$SCRIPT_DIR/gitea_ini.sh" +# shellcheck source=gitea_mirror_lib.sh +. "$SCRIPT_DIR/gitea_mirror_lib.sh" +# shellcheck source=gitea_submodule_lib.sh +. "$SCRIPT_DIR/gitea_submodule_lib.sh" + +GITEA_OWNER="${GITEA_OWNER:-admin}" +GITEA_REF="${GITEA_REF:-HEAD}" +GITEA_SUBMODULES_CONF="${GITEA_SUBMODULES_CONF:-$SCRIPT_DIR/submodules.defaults.conf}" + +log() { printf '[gitea-diagnose] %s\n' "$*"; } +warn() { printf '[gitea-diagnose] WARN: %s\n' "$*" >&2; } + +_arg="${1:-.}" +_repo_dir="" +_gitmodules="" + +case "$_arg" in + *.gitmodules) + _gitmodules="$(cd "$(dirname "$_arg")" && pwd)/$(basename "$_arg")" + _parent="$(dirname "$_gitmodules")" + [ -d "$_parent/.git" ] || [ -f "$_parent/.git" ] && _repo_dir="$(cd "$_parent" && pwd)" || _repo_dir="$_parent" + ;; + *) + [ -d "$_arg" ] || { warn "not a directory: $_arg"; exit 1; } + _repo_dir="$(cd "$_arg" && pwd)" + _gitmodules="$_repo_dir/.gitmodules" + ;; +esac + +gitea_load_config || exit 1 +gitea_ensure_cli_access || exit 1 + +log "ini=$GITEA_APP_INI owner=$GITEA_OWNER" +log "checkout=$_repo_dir" +log "gitmodules=$_gitmodules" + +if [ -r "$_gitmodules" ]; then + log ".gitmodules present ($(wc -l < "$_gitmodules" | tr -d ' ') lines)" +else + warn ".gitmodules MISSING on BE — submodule migrate cannot discover entries" +fi + +if command -v git >/dev/null 2>&1 && [ -d "$_repo_dir/.git" ] || [ -f "$_repo_dir/.git" ]; then + log "git branch: $(git -C "$_repo_dir" branch --show-current 2>/dev/null || echo '?')" + log "git HEAD: $(git -C "$_repo_dir" rev-parse HEAD 2>/dev/null || echo '?')" + for _p in third-party/libvpx third-party/opus third-party/speex; do + _sha="$(git -C "$_repo_dir" rev-parse "$GITEA_REF:$_p" 2>/dev/null)" || _sha="(no gitlink)" + log " $GITEA_REF:$_p -> $_sha" + done +else + warn "not a git checkout — only .gitmodules / defaults.conf can be used" +fi + +log "collect_entries (what migrate would mirror):" +gitea_submodule_collect_entries "$_repo_dir" "$GITEA_REF" "$_gitmodules" "$GITEA_SUBMODULES_CONF" \ + | while IFS= read -r _e; do + [ -n "$_e" ] && log " $_e" + done + +log "Gitea repos via API:" +if _names="$(gitea_list_repos_api "$GITEA_OWNER" 2>/dev/null)"; then + echo "$_names" | while read -r _r; do + [ -n "$_r" ] && log " $GITEA_OWNER/$_r" + done + log "count=$(echo "$_names" | grep -c . 2>/dev/null || echo 0)" +else + warn "API list failed" +fi + +log "on-disk bare repos:" +if [ -d "${GITEA_REPO_ROOT:-/var/lib/gitea/data/gitea-repositories}/$GITEA_OWNER" ]; then + ls -1 "${GITEA_REPO_ROOT}/$GITEA_OWNER" 2>/dev/null | while read -r _d; do + log " $GITEA_OWNER/$_d" + done +else + warn "no directory ${GITEA_REPO_ROOT}/$GITEA_OWNER" +fi + +cat <&2; } die() { printf '[gitea-migrate] ERROR: %s\n' "$*" >&2; exit 1; } -resolve_repo_dir() { +# Sets GITEA_REPO_DIR and GITEA_GITMODULES_FILE. +resolve_repo_inputs() { _arg="${1:-}" + GITEA_REPO_DIR="" + GITEA_GITMODULES_FILE="" + if [ -n "$_arg" ]; then - [ -d "$_arg" ] || die "not a directory: $_arg" - printf '%s' "$(cd "$_arg" && pwd)" - return 0 + case "$_arg" in + *.gitmodules) + [ -r "$_arg" ] || die "cannot read $_arg" + GITEA_GITMODULES_FILE="$(cd "$(dirname "$_arg")" && pwd)/$(basename "$_arg")" + _parent="$(dirname "$GITEA_GITMODULES_FILE")" + if [ -d "$_parent/.git" ] || [ -f "$_parent/.git" ]; then + GITEA_REPO_DIR="$(cd "$_parent" && pwd)" + else + GITEA_REPO_DIR="$_parent" + fi + return 0 + ;; + *) + [ -d "$_arg" ] || die "not a directory: $_arg" + GITEA_REPO_DIR="$(cd "$_arg" && pwd)" + GITEA_GITMODULES_FILE="$GITEA_REPO_DIR/.gitmodules" + return 0 + ;; + esac fi if [ -d "./.git" ] || [ -f "./.git" ]; then - pwd + GITEA_REPO_DIR="$(pwd)" + GITEA_GITMODULES_FILE="$GITEA_REPO_DIR/.gitmodules" return 0 fi - die "pass android_cast checkout path (needs .gitmodules and git history)" + die "pass android_cast checkout path or .gitmodules (needs .gitmodules)" } migrate_main_repo() { @@ -88,12 +110,15 @@ migrate_submodule() { log "submodule $_path ($GITEA_OWNER/$_repo)" log " upstream: $_upstream" - log " pinned @ $GITEA_REF: $_sha" - - if git ls-remote "$_upstream" "$_sha" >/dev/null 2>&1; then - log " commit on upstream" + if [ "$_sha" = "-" ]; then + log " pinned @ $GITEA_REF: (unknown — no gitlink in checkout; mirroring upstream anyway)" else - warn " commit $_sha not on upstream (mirror may still fetch via branch tip)" + log " pinned @ $GITEA_REF: $_sha" + if git ls-remote "$_upstream" "$_sha" >/dev/null 2>&1; then + log " commit on upstream" + else + warn " commit $_sha not on upstream (mirror may still fetch via branch tip)" + fi fi if ! gitea_mirror_pull_repo "$_upstream" "$_repo" "$GITEA_OWNER" \ @@ -106,6 +131,11 @@ migrate_submodule() { 2>/dev/null || true gitea_sync_mirror_api "$GITEA_OWNER" "$_repo" || true + if [ "$_sha" = "-" ]; then + log " OK mirror -> $(gitea_public_repo_url_for "$GITEA_OWNER" "$_repo")" + return 0 + fi + if gitea_mirror_has_commit "$GITEA_OWNER" "$_repo" "$_sha"; then log " OK pinned commit present -> $(gitea_public_repo_url_for "$GITEA_OWNER" "$_repo")" return 0 @@ -115,6 +145,23 @@ migrate_submodule() { return 2 } +print_gitea_repo_inventory() { + log "Gitea repos for $GITEA_OWNER (API):" + _names="$(gitea_list_repos_api "$GITEA_OWNER" 2>/dev/null)" || _names="" + if [ -z "$_names" ]; then + warn " could not list repos via API (token/CLI)" + return 1 + fi + _n=0 + echo "$_names" | while read -r _r; do + [ -n "$_r" ] || continue + log " - $GITEA_OWNER/$_r" + _n=$((_n + 1)) + done + _count="$(echo "$_names" | grep -c . 2>/dev/null || echo 0)" + log "total listed: $_count (expect android_cast + libvpx + opus + speex)" +} + print_fast_sync_guide() { _sync_sh="$SCRIPT_DIR/gitea_sync_mirror.sh" _hook="$SCRIPT_DIR/fe-android_cast-post-receive.example" @@ -216,15 +263,20 @@ else fi if [ "$GITEA_SKIP_SUBMODULES" -eq 0 ]; then - _repo_dir="$(resolve_repo_dir "$_repo_arg")" - log "discovering external submodules in $_repo_dir" + resolve_repo_inputs "$_repo_arg" + log "submodule inputs: repo=$GITEA_REPO_DIR gitmodules=$GITEA_GITMODULES_FILE ref=$GITEA_REF" _sub_list="$WORK_DIR/submodules.list" - : > "$_sub_list" - gitea_submodule_discover_external "$_repo_dir" "$GITEA_REF" > "$_sub_list" || true + gitea_submodule_collect_entries "$GITEA_REPO_DIR" "$GITEA_REF" \ + "$GITEA_GITMODULES_FILE" "$GITEA_SUBMODULES_CONF" > "$_sub_list" || true if [ ! -s "$_sub_list" ]; then - warn "no external submodules at $GITEA_REF (empty .gitmodules or only local URLs)" - else - while IFS= read -r _line; do + die "no external submodules found — check .gitmodules on BE or $GITEA_SUBMODULES_CONF" + fi + log "submodules to mirror:" + while IFS= read -r _preview; do + [ -n "$_preview" ] || continue + log " $_preview" + done < "$_sub_list" + while IFS= read -r _line; do [ -n "$_line" ] || continue gitea_submodule_entry_valid "$_line" || continue _rc=0 @@ -235,12 +287,14 @@ if [ "$GITEA_SKIP_SUBMODULES" -eq 0 ]; then *) _sub_fail=$((_sub_fail + 1)) ;; esac done < "$_sub_list" - fi else log "skipping submodule mirrors (--skip-submodules)" fi log "summary: main ok=$_main_ok fail=$_main_fail | submodules ok=$_sub_ok warn=$_sub_warn fail=$_sub_fail" +print_gitea_repo_inventory print_fast_sync_guide -[ "$_main_fail" -eq 0 ] && [ "$_sub_fail" -eq 0 ] +_sub_migrated=$((_sub_ok + _sub_warn)) +[ "$_main_fail" -eq 0 ] && [ "$_sub_fail" -eq 0 ] \ + && { [ "$GITEA_SKIP_SUBMODULES" -eq 1 ] || [ "$_sub_migrated" -gt 0 ]; } diff --git a/examples/crash_reporter/backend/scripts/gitea/gitea_mirror_lib.sh b/examples/crash_reporter/backend/scripts/gitea/gitea_mirror_lib.sh index a23099c..0672c1f 100755 --- a/examples/crash_reporter/backend/scripts/gitea/gitea_mirror_lib.sh +++ b/examples/crash_reporter/backend/scripts/gitea/gitea_mirror_lib.sh @@ -115,6 +115,11 @@ EOF if [ "$_http" = "409" ] || [ "$_http" = "422" ]; then return 2 fi + if [ -r "/tmp/gitea-migrate-$$.json" ]; then + printf '[gitea-mirror] migrate HTTP %s: ' "$_http" >&2 + head -c 400 "/tmp/gitea-migrate-$$.json" >&2 || true + printf '\n' >&2 + fi return 1 } @@ -137,6 +142,30 @@ gitea_mirror_push_git() { return 0 } +gitea_repo_exists_api() { + _owner="$1" + _repo="$2" + command -v curl >/dev/null 2>&1 || return 1 + _token="${GITEA_ADMIN_TOKEN:-}" + [ -n "$_token" ] || _token="$(gitea_resolve_push_token)" || return 1 + _api="$(gitea_api_base)" + _http="$(curl -sS -o /dev/null -w '%{http_code}' \ + "$_api/repos/$_owner/$_repo" \ + -H "Authorization: token $_token")" + [ "$_http" = "200" ] +} + +gitea_list_repos_api() { + _owner="$1" + command -v curl >/dev/null 2>&1 || return 1 + _token="${GITEA_ADMIN_TOKEN:-}" + [ -n "$_token" ] || _token="$(gitea_resolve_push_token)" || return 1 + _api="$(gitea_api_base)" + curl -sS "$_api/users/$_owner/repos?limit=50" \ + -H "Authorization: token $_token" \ + | awk -F'"' '/"name":/ { print $4 }' +} + gitea_sync_mirror_api() { _owner="$1" _repo="$2" @@ -192,9 +221,13 @@ gitea_mirror_pull_repo() { && return 0 _api_rc=$? if [ "$_api_rc" -eq 2 ]; then - gitea_mirror_set_interval_api "$_owner" "$_repo" "$_interval" 2>/dev/null || true - gitea_sync_mirror_api "$_owner" "$_repo" && return 0 - return 0 + if gitea_repo_exists_api "$_owner" "$_repo"; then + gitea_mirror_set_interval_api "$_owner" "$_repo" "$_interval" 2>/dev/null || true + gitea_sync_mirror_api "$_owner" "$_repo" && return 0 + return 0 + fi + printf '[gitea-mirror] migrate rejected for %s/%s but repo missing — trying git push\n' \ + "$_owner" "$_repo" >&2 fi mkdir -p "$_work_dir" gitea_mirror_push_git "$_owner" "$_repo" "$_clone_url" "$_work_dir" || _rc=1 diff --git a/examples/crash_reporter/backend/scripts/gitea/gitea_submodule_lib.sh b/examples/crash_reporter/backend/scripts/gitea/gitea_submodule_lib.sh index c24fa02..41a9ab5 100644 --- a/examples/crash_reporter/backend/scripts/gitea/gitea_submodule_lib.sh +++ b/examples/crash_reporter/backend/scripts/gitea/gitea_submodule_lib.sh @@ -61,14 +61,11 @@ gitea_submodule_parse_gitmodules() { ' "$_file" } -# repo|url|path|sha[|branch] for each external submodule with a gitlink at REF. -gitea_submodule_discover_external() { - _repo_dir="$1" - _ref="${2:-HEAD}" - _gitmodules="$_repo_dir/.gitmodules" - - command -v git >/dev/null 2>&1 || return 1 - git -C "$_repo_dir" rev-parse --verify "$_ref" >/dev/null 2>&1 || return 1 +# repo|url|path|sha[|branch] — external entries from .gitmodules (sha may be "-" if unknown). +gitea_submodule_discover_from_gitmodules() { + _gitmodules="$1" + _repo_dir="${2:-}" + _ref="${3:-HEAD}" [ -r "$_gitmodules" ] || return 1 gitea_submodule_parse_gitmodules "$_gitmodules" | while IFS= read -r _line; do @@ -85,10 +82,15 @@ gitea_submodule_discover_external() { gitea_submodule_is_external_url "$_url" || continue - _sha="$(git -C "$_repo_dir" rev-parse "$_ref:$_path" 2>/dev/null)" || continue - case "$_sha" in - ??000000000000000000000000000000000000) continue ;; - esac + _sha="-" + if [ -n "$_repo_dir" ] && command -v git >/dev/null 2>&1 \ + && git -C "$_repo_dir" rev-parse --verify "$_ref" >/dev/null 2>&1; then + _got="$(git -C "$_repo_dir" rev-parse "$_ref:$_path" 2>/dev/null)" || _got="" + case "$_got" in + ??000000000000000000000000000000000000) _sha="-" ;; + ????????????????????????????????????????*) _sha="$_got" ;; + esac + fi if [ -n "$_branch" ]; then printf '%s|%s|%s|%s|%s\n' "$_repo" "$_url" "$_path" "$_sha" "$_branch" @@ -98,6 +100,86 @@ gitea_submodule_discover_external() { done } +# repo|url|path|sha[|branch] for each external submodule with a gitlink at REF. +gitea_submodule_discover_external() { + _repo_dir="$1" + _ref="${2:-HEAD}" + _gitmodules="$_repo_dir/.gitmodules" + + command -v git >/dev/null 2>&1 || return 1 + git -C "$_repo_dir" rev-parse --verify "$_ref" >/dev/null 2>&1 || return 1 + [ -r "$_gitmodules" ] || return 1 + + gitea_submodule_discover_from_gitmodules "$_gitmodules" "$_repo_dir" "$_ref" \ + | while IFS= read -r _line; do + _sha="${_line##*|}" + case "$_sha" in + -) continue ;; + esac + printf '%s\n' "$_line" + done +} + +# repo|url|path|sha[|branch] from submodules.defaults.conf (sha resolved when possible). +gitea_submodule_load_defaults() { + _conf="$1" + _repo_dir="${2:-}" + _ref="${3:-HEAD}" + [ -r "$_conf" ] || return 1 + + grep -v '^[[:space:]]*#' "$_conf" | grep -v '^[[:space:]]*$' | while IFS= read -r _line; do + [ -n "$_line" ] || continue + _repo="${_line%%|*}" + _rest="${_line#*|}" + _url="${_rest%%|*}" + _rest2="${_rest#*|}" + _path="${_rest2%%|*}" + _branch="${_rest2#*|}" + case "$_branch" in + "$_path") _branch="" ;; + esac + gitea_submodule_is_external_url "$_url" || continue + _sha="-" + if [ -n "$_repo_dir" ] && command -v git >/dev/null 2>&1 \ + && git -C "$_repo_dir" rev-parse --verify "$_ref" >/dev/null 2>&1; then + _got="$(git -C "$_repo_dir" rev-parse "$_ref:$_path" 2>/dev/null)" || _got="" + case "$_got" in + ????????????????????????????????????????*) _sha="$_got" ;; + esac + fi + if [ -n "$_branch" ]; then + printf '%s|%s|%s|%s|%s\n' "$_repo" "$_url" "$_path" "$_sha" "$_branch" + else + printf '%s|%s|%s|%s\n' "$_repo" "$_url" "$_path" "$_sha" + fi + done +} + +# Merge discovery sources; first win per repo name. stdout: repo|url|path|sha[|branch] +gitea_submodule_collect_entries() { + _repo_dir="${1:-}" + _ref="${2:-HEAD}" + _gitmodules="${3:-}" + _defaults="${4:-}" + + _work="$(mktemp /tmp/gitea-submods-XXXXXX)" + + if [ -n "$_gitmodules" ] && [ -r "$_gitmodules" ]; then + gitea_submodule_discover_from_gitmodules "$_gitmodules" "$_repo_dir" "$_ref" >> "$_work" 2>/dev/null || true + elif [ -n "$_repo_dir" ] && [ -r "$_repo_dir/.gitmodules" ]; then + gitea_submodule_discover_from_gitmodules "$_repo_dir/.gitmodules" "$_repo_dir" "$_ref" >> "$_work" 2>/dev/null || true + fi + + if [ ! -s "$_work" ] && [ -n "$_defaults" ] && [ -r "$_defaults" ]; then + gitea_submodule_load_defaults "$_defaults" "$_repo_dir" "$_ref" >> "$_work" 2>/dev/null || true + elif [ -n "$_defaults" ] && [ -r "$_defaults" ]; then + gitea_submodule_load_defaults "$_defaults" "$_repo_dir" "$_ref" >> "$_work" 2>/dev/null || true + fi + + awk -F'|' '!seen[$1]++' "$_work" + rm -f "$_work" +} + gitea_submodule_entry_valid() { case "$1" in *'|'*) ;;