gitea struggle

2026-07-29 05:17:39 +03:00 · 2026-06-10 22:15:17 +02:00
parent 89a0365038
commit 91cac577e8
5 changed files with 327 additions and 35 deletions
--- a/examples/crash_reporter/backend/scripts/gitea/README.md
+++ b/examples/crash_reporter/backend/scripts/gitea/README.md
@@ -178,6 +178,24 @@ sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh .
 GITEA_REF=next sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh .
 ```

+### Troubleshooting: only `android_cast` visible (Repositories 1)
+
+`gitea_attach_remote.sh` mirrors **only the main repo**. Submodule repos need a separate step.
+
+```bash
+# 1) See what BE would mirror / what exists in Gitea:
+sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_diagnose_mirrors.sh .
+
+# 2) Create submodule mirrors (main already exists):
+sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh . --skip-main
+# or:
+sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_mirror_submodules.sh .gitmodules
+```
+
+Expect **4 repos** under `admin`: `android_cast`, `libvpx`, `opus`, `speex`. If migrate summary shows `submodules ok=0` and a WARN about empty discovery, `.gitmodules` is missing on the BE checkout — the updated migrate script falls back to `submodules.defaults.conf`.
+
+In Gitea UI, use **Explore** or **Site Administration → Repositories** if the profile counter hides mirror repos.
+
 ### Fast sync for local `android_cast` mirror only

 Gitea **default minimum** pull interval is **10m**. To use **1m** for the FE git mirror:
--- a/examples/crash_reporter/backend/scripts/gitea/gitea_diagnose_mirrors.sh
+++ b/examples/crash_reporter/backend/scripts/gitea/gitea_diagnose_mirrors.sh
@@ -0,0 +1,105 @@
+#!/bin/ash
+#
+# Debug why submodule mirrors are missing in Gitea UI.
+#
+# Usage:
+#   sudo ./gitea_diagnose_mirrors.sh
+#   sudo ./gitea_diagnose_mirrors.sh /var/www/.../android_cast
+#   sudo ./gitea_diagnose_mirrors.sh /var/www/.../android_cast/.gitmodules
+#
+set -eu
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck source=gitea_ini.sh
+. "$SCRIPT_DIR/gitea_ini.sh"
+# shellcheck source=gitea_mirror_lib.sh
+. "$SCRIPT_DIR/gitea_mirror_lib.sh"
+# shellcheck source=gitea_submodule_lib.sh
+. "$SCRIPT_DIR/gitea_submodule_lib.sh"
+
+GITEA_OWNER="${GITEA_OWNER:-admin}"
+GITEA_REF="${GITEA_REF:-HEAD}"
+GITEA_SUBMODULES_CONF="${GITEA_SUBMODULES_CONF:-$SCRIPT_DIR/submodules.defaults.conf}"
+
+log() { printf '[gitea-diagnose] %s\n' "$*"; }
+warn() { printf '[gitea-diagnose] WARN: %s\n' "$*" >&2; }
+
+_arg="${1:-.}"
+_repo_dir=""
+_gitmodules=""
+
+case "$_arg" in
+  *.gitmodules)
+    _gitmodules="$(cd "$(dirname "$_arg")" && pwd)/$(basename "$_arg")"
+    _parent="$(dirname "$_gitmodules")"
+    [ -d "$_parent/.git" ] || [ -f "$_parent/.git" ] && _repo_dir="$(cd "$_parent" && pwd)" || _repo_dir="$_parent"
+    ;;
+  *)
+    [ -d "$_arg" ] || { warn "not a directory: $_arg"; exit 1; }
+    _repo_dir="$(cd "$_arg" && pwd)"
+    _gitmodules="$_repo_dir/.gitmodules"
+    ;;
+esac
+
+gitea_load_config || exit 1
+gitea_ensure_cli_access || exit 1
+
+log "ini=$GITEA_APP_INI  owner=$GITEA_OWNER"
+log "checkout=$_repo_dir"
+log "gitmodules=$_gitmodules"
+
+if [ -r "$_gitmodules" ]; then
+  log ".gitmodules present ($(wc -l < "$_gitmodules" | tr -d ' ') lines)"
+else
+  warn ".gitmodules MISSING on BE — submodule migrate cannot discover entries"
+fi
+
+if command -v git >/dev/null 2>&1 && [ -d "$_repo_dir/.git" ] || [ -f "$_repo_dir/.git" ]; then
+  log "git branch: $(git -C "$_repo_dir" branch --show-current 2>/dev/null || echo '?')"
+  log "git HEAD: $(git -C "$_repo_dir" rev-parse HEAD 2>/dev/null || echo '?')"
+  for _p in third-party/libvpx third-party/opus third-party/speex; do
+    _sha="$(git -C "$_repo_dir" rev-parse "$GITEA_REF:$_p" 2>/dev/null)" || _sha="(no gitlink)"
+    log "  $GITEA_REF:$_p -> $_sha"
+  done
+else
+  warn "not a git checkout — only .gitmodules / defaults.conf can be used"
+fi
+
+log "collect_entries (what migrate would mirror):"
+gitea_submodule_collect_entries "$_repo_dir" "$GITEA_REF" "$_gitmodules" "$GITEA_SUBMODULES_CONF" \
+  | while IFS= read -r _e; do
+    [ -n "$_e" ] && log "  $_e"
+  done
+
+log "Gitea repos via API:"
+if _names="$(gitea_list_repos_api "$GITEA_OWNER" 2>/dev/null)"; then
+  echo "$_names" | while read -r _r; do
+    [ -n "$_r" ] && log "  $GITEA_OWNER/$_r"
+  done
+  log "count=$(echo "$_names" | grep -c . 2>/dev/null || echo 0)"
+else
+  warn "API list failed"
+fi
+
+log "on-disk bare repos:"
+if [ -d "${GITEA_REPO_ROOT:-/var/lib/gitea/data/gitea-repositories}/$GITEA_OWNER" ]; then
+  ls -1 "${GITEA_REPO_ROOT}/$GITEA_OWNER" 2>/dev/null | while read -r _d; do
+    log "  $GITEA_OWNER/$_d"
+  done
+else
+  warn "no directory ${GITEA_REPO_ROOT}/$GITEA_OWNER"
+fi
+
+cat <<EOF
+
+If count=1 (only android_cast):
+  • gitea_attach_remote.sh only mirrors the main repo — run submodule step:
+      sudo ./gitea_migrate_project.sh . --skip-main
+    or:
+      sudo ./gitea_mirror_submodules.sh .gitmodules
+  • If collect_entries is empty: sync .gitmodules to BE or use submodules.defaults.conf
+
+Gitea UI: profile "Repositories N" may exclude mirrors on some versions —
+  try Explore → search libvpx, or Site Administration → Repositories.
+
+EOF
--- a/examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh
+++ b/examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh
@@ -32,24 +32,46 @@ GITEA_PRIVATE="${GITEA_PRIVATE:-false}"
 GITEA_REF="${GITEA_REF:-HEAD}"
 GITEA_SKIP_MAIN="${GITEA_SKIP_MAIN:-0}"
 GITEA_SKIP_SUBMODULES="${GITEA_SKIP_SUBMODULES:-0}"
+GITEA_SUBMODULES_CONF="${GITEA_SUBMODULES_CONF:-$SCRIPT_DIR/submodules.defaults.conf}"
 WORK_DIR="${WORK_DIR:-/tmp/gitea-migrate-$$}"

 log() { printf '[gitea-migrate] %s\n' "$*"; }
 warn() { printf '[gitea-migrate] WARN: %s\n' "$*" >&2; }
 die() { printf '[gitea-migrate] ERROR: %s\n' "$*" >&2; exit 1; }

-resolve_repo_dir() {
+# Sets GITEA_REPO_DIR and GITEA_GITMODULES_FILE.
+resolve_repo_inputs() {
  _arg="${1:-}"
+  GITEA_REPO_DIR=""
+  GITEA_GITMODULES_FILE=""
+
  if [ -n "$_arg" ]; then
-    [ -d "$_arg" ] || die "not a directory: $_arg"
-    printf '%s' "$(cd "$_arg" && pwd)"
-    return 0
+    case "$_arg" in
+      *.gitmodules)
+        [ -r "$_arg" ] || die "cannot read $_arg"
+        GITEA_GITMODULES_FILE="$(cd "$(dirname "$_arg")" && pwd)/$(basename "$_arg")"
+        _parent="$(dirname "$GITEA_GITMODULES_FILE")"
+        if [ -d "$_parent/.git" ] || [ -f "$_parent/.git" ]; then
+          GITEA_REPO_DIR="$(cd "$_parent" && pwd)"
+        else
+          GITEA_REPO_DIR="$_parent"
+        fi
+        return 0
+        ;;
+      *)
+        [ -d "$_arg" ] || die "not a directory: $_arg"
+        GITEA_REPO_DIR="$(cd "$_arg" && pwd)"
+        GITEA_GITMODULES_FILE="$GITEA_REPO_DIR/.gitmodules"
+        return 0
+        ;;
+    esac
  fi
  if [ -d "./.git" ] || [ -f "./.git" ]; then
-    pwd
+    GITEA_REPO_DIR="$(pwd)"
+    GITEA_GITMODULES_FILE="$GITEA_REPO_DIR/.gitmodules"
    return 0
  fi
-  die "pass android_cast checkout path (needs .gitmodules and git history)"
+  die "pass android_cast checkout path or .gitmodules (needs .gitmodules)"
 }

 migrate_main_repo() {
@@ -88,12 +110,15 @@ migrate_submodule() {

  log "submodule $_path ($GITEA_OWNER/$_repo)"
  log "  upstream: $_upstream"
-  log "  pinned @ $GITEA_REF: $_sha"
-
-  if git ls-remote "$_upstream" "$_sha" >/dev/null 2>&1; then
-    log "  commit on upstream"
+  if [ "$_sha" = "-" ]; then
+    log "  pinned @ $GITEA_REF: (unknown — no gitlink in checkout; mirroring upstream anyway)"
  else
-    warn "  commit $_sha not on upstream (mirror may still fetch via branch tip)"
+    log "  pinned @ $GITEA_REF: $_sha"
+    if git ls-remote "$_upstream" "$_sha" >/dev/null 2>&1; then
+      log "  commit on upstream"
+    else
+      warn "  commit $_sha not on upstream (mirror may still fetch via branch tip)"
+    fi
  fi

  if ! gitea_mirror_pull_repo "$_upstream" "$_repo" "$GITEA_OWNER" \
@@ -106,6 +131,11 @@ migrate_submodule() {
    2>/dev/null || true
  gitea_sync_mirror_api "$GITEA_OWNER" "$_repo" || true

+  if [ "$_sha" = "-" ]; then
+    log "  OK mirror -> $(gitea_public_repo_url_for "$GITEA_OWNER" "$_repo")"
+    return 0
+  fi
+
  if gitea_mirror_has_commit "$GITEA_OWNER" "$_repo" "$_sha"; then
    log "  OK pinned commit present -> $(gitea_public_repo_url_for "$GITEA_OWNER" "$_repo")"
    return 0
@@ -115,6 +145,23 @@ migrate_submodule() {
  return 2
 }

+print_gitea_repo_inventory() {
+  log "Gitea repos for $GITEA_OWNER (API):"
+  _names="$(gitea_list_repos_api "$GITEA_OWNER" 2>/dev/null)" || _names=""
+  if [ -z "$_names" ]; then
+    warn "  could not list repos via API (token/CLI)"
+    return 1
+  fi
+  _n=0
+  echo "$_names" | while read -r _r; do
+    [ -n "$_r" ] || continue
+    log "  - $GITEA_OWNER/$_r"
+    _n=$((_n + 1))
+  done
+  _count="$(echo "$_names" | grep -c . 2>/dev/null || echo 0)"
+  log "total listed: $_count (expect android_cast + libvpx + opus + speex)"
+}
+
 print_fast_sync_guide() {
  _sync_sh="$SCRIPT_DIR/gitea_sync_mirror.sh"
  _hook="$SCRIPT_DIR/fe-android_cast-post-receive.example"
@@ -216,15 +263,20 @@ else
 fi

 if [ "$GITEA_SKIP_SUBMODULES" -eq 0 ]; then
-  _repo_dir="$(resolve_repo_dir "$_repo_arg")"
-  log "discovering external submodules in $_repo_dir"
+  resolve_repo_inputs "$_repo_arg"
+  log "submodule inputs: repo=$GITEA_REPO_DIR  gitmodules=$GITEA_GITMODULES_FILE  ref=$GITEA_REF"
  _sub_list="$WORK_DIR/submodules.list"
-  : > "$_sub_list"
-  gitea_submodule_discover_external "$_repo_dir" "$GITEA_REF" > "$_sub_list" || true
+  gitea_submodule_collect_entries "$GITEA_REPO_DIR" "$GITEA_REF" \
+    "$GITEA_GITMODULES_FILE" "$GITEA_SUBMODULES_CONF" > "$_sub_list" || true
  if [ ! -s "$_sub_list" ]; then
-    warn "no external submodules at $GITEA_REF (empty .gitmodules or only local URLs)"
-  else
-    while IFS= read -r _line; do
+    die "no external submodules found — check .gitmodules on BE or $GITEA_SUBMODULES_CONF"
+  fi
+  log "submodules to mirror:"
+  while IFS= read -r _preview; do
+    [ -n "$_preview" ] || continue
+    log "  $_preview"
+  done < "$_sub_list"
+  while IFS= read -r _line; do
      [ -n "$_line" ] || continue
      gitea_submodule_entry_valid "$_line" || continue
      _rc=0
@@ -235,12 +287,14 @@ if [ "$GITEA_SKIP_SUBMODULES" -eq 0 ]; then
        *) _sub_fail=$((_sub_fail + 1)) ;;
      esac
    done < "$_sub_list"
-  fi
 else
  log "skipping submodule mirrors (--skip-submodules)"
 fi

 log "summary: main ok=$_main_ok fail=$_main_fail | submodules ok=$_sub_ok warn=$_sub_warn fail=$_sub_fail"
+print_gitea_repo_inventory
 print_fast_sync_guide

-[ "$_main_fail" -eq 0 ] && [ "$_sub_fail" -eq 0 ]
+_sub_migrated=$((_sub_ok + _sub_warn))
+[ "$_main_fail" -eq 0 ] && [ "$_sub_fail" -eq 0 ] \
+  && { [ "$GITEA_SKIP_SUBMODULES" -eq 1 ] || [ "$_sub_migrated" -gt 0 ]; }
--- a/examples/crash_reporter/backend/scripts/gitea/gitea_mirror_lib.sh
+++ b/examples/crash_reporter/backend/scripts/gitea/gitea_mirror_lib.sh
@@ -115,6 +115,11 @@ EOF
  if [ "$_http" = "409" ] || [ "$_http" = "422" ]; then
    return 2
  fi
+  if [ -r "/tmp/gitea-migrate-$$.json" ]; then
+    printf '[gitea-mirror] migrate HTTP %s: ' "$_http" >&2
+    head -c 400 "/tmp/gitea-migrate-$$.json" >&2 || true
+    printf '\n' >&2
+  fi
  return 1
 }

@@ -137,6 +142,30 @@ gitea_mirror_push_git() {
  return 0
 }

+gitea_repo_exists_api() {
+  _owner="$1"
+  _repo="$2"
+  command -v curl >/dev/null 2>&1 || return 1
+  _token="${GITEA_ADMIN_TOKEN:-}"
+  [ -n "$_token" ] || _token="$(gitea_resolve_push_token)" || return 1
+  _api="$(gitea_api_base)"
+  _http="$(curl -sS -o /dev/null -w '%{http_code}' \
+    "$_api/repos/$_owner/$_repo" \
+    -H "Authorization: token $_token")"
+  [ "$_http" = "200" ]
+}
+
+gitea_list_repos_api() {
+  _owner="$1"
+  command -v curl >/dev/null 2>&1 || return 1
+  _token="${GITEA_ADMIN_TOKEN:-}"
+  [ -n "$_token" ] || _token="$(gitea_resolve_push_token)" || return 1
+  _api="$(gitea_api_base)"
+  curl -sS "$_api/users/$_owner/repos?limit=50" \
+    -H "Authorization: token $_token" \
+    | awk -F'"' '/"name":/ { print $4 }'
+}
+
 gitea_sync_mirror_api() {
  _owner="$1"
  _repo="$2"
@@ -192,9 +221,13 @@ gitea_mirror_pull_repo() {
    && return 0
  _api_rc=$?
  if [ "$_api_rc" -eq 2 ]; then
-    gitea_mirror_set_interval_api "$_owner" "$_repo" "$_interval" 2>/dev/null || true
-    gitea_sync_mirror_api "$_owner" "$_repo" && return 0
-    return 0
+    if gitea_repo_exists_api "$_owner" "$_repo"; then
+      gitea_mirror_set_interval_api "$_owner" "$_repo" "$_interval" 2>/dev/null || true
+      gitea_sync_mirror_api "$_owner" "$_repo" && return 0
+      return 0
+    fi
+    printf '[gitea-mirror] migrate rejected for %s/%s but repo missing — trying git push\n' \
+      "$_owner" "$_repo" >&2
  fi
  mkdir -p "$_work_dir"
  gitea_mirror_push_git "$_owner" "$_repo" "$_clone_url" "$_work_dir" || _rc=1
--- a/examples/crash_reporter/backend/scripts/gitea/gitea_submodule_lib.sh
+++ b/examples/crash_reporter/backend/scripts/gitea/gitea_submodule_lib.sh
@@ -61,14 +61,11 @@ gitea_submodule_parse_gitmodules() {
  ' "$_file"
 }

-# repo|url|path|sha[|branch] for each external submodule with a gitlink at REF.
-gitea_submodule_discover_external() {
-  _repo_dir="$1"
-  _ref="${2:-HEAD}"
-  _gitmodules="$_repo_dir/.gitmodules"
-
-  command -v git >/dev/null 2>&1 || return 1
-  git -C "$_repo_dir" rev-parse --verify "$_ref" >/dev/null 2>&1 || return 1
+# repo|url|path|sha[|branch] — external entries from .gitmodules (sha may be "-" if unknown).
+gitea_submodule_discover_from_gitmodules() {
+  _gitmodules="$1"
+  _repo_dir="${2:-}"
+  _ref="${3:-HEAD}"
  [ -r "$_gitmodules" ] || return 1

  gitea_submodule_parse_gitmodules "$_gitmodules" | while IFS= read -r _line; do
@@ -85,10 +82,15 @@ gitea_submodule_discover_external() {

    gitea_submodule_is_external_url "$_url" || continue

-    _sha="$(git -C "$_repo_dir" rev-parse "$_ref:$_path" 2>/dev/null)" || continue
-    case "$_sha" in
-      ??000000000000000000000000000000000000) continue ;;
-    esac
+    _sha="-"
+    if [ -n "$_repo_dir" ] && command -v git >/dev/null 2>&1 \
+      && git -C "$_repo_dir" rev-parse --verify "$_ref" >/dev/null 2>&1; then
+      _got="$(git -C "$_repo_dir" rev-parse "$_ref:$_path" 2>/dev/null)" || _got=""
+      case "$_got" in
+        ??000000000000000000000000000000000000) _sha="-" ;;
+        ????????????????????????????????????????*) _sha="$_got" ;;
+      esac
+    fi

    if [ -n "$_branch" ]; then
      printf '%s|%s|%s|%s|%s\n' "$_repo" "$_url" "$_path" "$_sha" "$_branch"
@@ -98,6 +100,86 @@ gitea_submodule_discover_external() {
  done
 }

+# repo|url|path|sha[|branch] for each external submodule with a gitlink at REF.
+gitea_submodule_discover_external() {
+  _repo_dir="$1"
+  _ref="${2:-HEAD}"
+  _gitmodules="$_repo_dir/.gitmodules"
+
+  command -v git >/dev/null 2>&1 || return 1
+  git -C "$_repo_dir" rev-parse --verify "$_ref" >/dev/null 2>&1 || return 1
+  [ -r "$_gitmodules" ] || return 1
+
+  gitea_submodule_discover_from_gitmodules "$_gitmodules" "$_repo_dir" "$_ref" \
+    | while IFS= read -r _line; do
+      _sha="${_line##*|}"
+      case "$_sha" in
+        -) continue ;;
+      esac
+      printf '%s\n' "$_line"
+    done
+}
+
+# repo|url|path|sha[|branch] from submodules.defaults.conf (sha resolved when possible).
+gitea_submodule_load_defaults() {
+  _conf="$1"
+  _repo_dir="${2:-}"
+  _ref="${3:-HEAD}"
+  [ -r "$_conf" ] || return 1
+
+  grep -v '^[[:space:]]*#' "$_conf" | grep -v '^[[:space:]]*$' | while IFS= read -r _line; do
+    [ -n "$_line" ] || continue
+    _repo="${_line%%|*}"
+    _rest="${_line#*|}"
+    _url="${_rest%%|*}"
+    _rest2="${_rest#*|}"
+    _path="${_rest2%%|*}"
+    _branch="${_rest2#*|}"
+    case "$_branch" in
+      "$_path") _branch="" ;;
+    esac
+    gitea_submodule_is_external_url "$_url" || continue
+    _sha="-"
+    if [ -n "$_repo_dir" ] && command -v git >/dev/null 2>&1 \
+      && git -C "$_repo_dir" rev-parse --verify "$_ref" >/dev/null 2>&1; then
+      _got="$(git -C "$_repo_dir" rev-parse "$_ref:$_path" 2>/dev/null)" || _got=""
+      case "$_got" in
+        ????????????????????????????????????????*) _sha="$_got" ;;
+      esac
+    fi
+    if [ -n "$_branch" ]; then
+      printf '%s|%s|%s|%s|%s\n' "$_repo" "$_url" "$_path" "$_sha" "$_branch"
+    else
+      printf '%s|%s|%s|%s\n' "$_repo" "$_url" "$_path" "$_sha"
+    fi
+  done
+}
+
+# Merge discovery sources; first win per repo name. stdout: repo|url|path|sha[|branch]
+gitea_submodule_collect_entries() {
+  _repo_dir="${1:-}"
+  _ref="${2:-HEAD}"
+  _gitmodules="${3:-}"
+  _defaults="${4:-}"
+
+  _work="$(mktemp /tmp/gitea-submods-XXXXXX)"
+
+  if [ -n "$_gitmodules" ] && [ -r "$_gitmodules" ]; then
+    gitea_submodule_discover_from_gitmodules "$_gitmodules" "$_repo_dir" "$_ref" >> "$_work" 2>/dev/null || true
+  elif [ -n "$_repo_dir" ] && [ -r "$_repo_dir/.gitmodules" ]; then
+    gitea_submodule_discover_from_gitmodules "$_repo_dir/.gitmodules" "$_repo_dir" "$_ref" >> "$_work" 2>/dev/null || true
+  fi
+
+  if [ ! -s "$_work" ] && [ -n "$_defaults" ] && [ -r "$_defaults" ]; then
+    gitea_submodule_load_defaults "$_defaults" "$_repo_dir" "$_ref" >> "$_work" 2>/dev/null || true
+  elif [ -n "$_defaults" ] && [ -r "$_defaults" ]; then
+    gitea_submodule_load_defaults "$_defaults" "$_repo_dir" "$_ref" >> "$_work" 2>/dev/null || true
+  fi
+
+  awk -F'|' '!seen[$1]++' "$_work"
+  rm -f "$_work"
+}
+
 gitea_submodule_entry_valid() {
  case "$1" in
    *'|'*) ;;