diff --git a/docs/INFRA.md b/docs/INFRA.md index eead148..a3f963a 100644 --- a/docs/INFRA.md +++ b/docs/INFRA.md @@ -232,4 +232,5 @@ Expect **200** (or **302** → crashes only if you intentionally redirect root). | Date | Note | |------|------| +| 2026-06-04 | Edge router WAN probe: `examples/network/wan-snapshot.sh` (world ↔ 134.17.26.161 vs 10.7.0.10) | | 2026-05-21 | Documented BE-only hub on port 80; 8089 = Janus only; FE `apps` vs `artc0` split | diff --git a/examples/crash_reporter/backend/scripts/gitea/README.md b/examples/crash_reporter/backend/scripts/gitea/README.md index 64a17ef..d36bf15 100644 --- a/examples/crash_reporter/backend/scripts/gitea/README.md +++ b/examples/crash_reporter/backend/scripts/gitea/README.md @@ -6,6 +6,18 @@ Scripts for the BE Gitea instance at `https://apps.f0xx.org/app/androidcast_proj `/var/www/localhost/htdocs/apps/app/androidcast_project/android_cast` is **never** modified. Only Gitea-owned paths from `/etc/gitea/app.ini` are backed up or reset. +## Temp files (/tmp) + +Gitea scripts use `mktemp` work dirs for `git clone --mirror` fallbacks. Each `*.mirror.git` tree is **removed after every repo** (success or failure) and the work dir is removed on script exit. + +Stale dirs (e.g. after `kill -9`) are pruned automatically on the next run (default: older than **60 minutes**). Manual cleanup: + +```bash +sudo ./examples/crash_reporter/backend/scripts/gitea/gitea_cleanup_tmp.sh +# immediate wipe of all gitea-* temps: +GITEA_TMP_MAX_AGE_MIN=0 sudo ./gitea_cleanup_tmp.sh +``` + ## Prerequisites (Alpine BE) - Gitea installed (`apk add gitea`), config at `/etc/gitea/app.ini` diff --git a/examples/crash_reporter/backend/scripts/gitea/gitea_attach_remote.sh b/examples/crash_reporter/backend/scripts/gitea/gitea_attach_remote.sh index 1c77a10..beef037 100755 --- a/examples/crash_reporter/backend/scripts/gitea/gitea_attach_remote.sh +++ b/examples/crash_reporter/backend/scripts/gitea/gitea_attach_remote.sh @@ -30,7 +30,7 @@ GITEA_OWNER="${GITEA_OWNER:-${GITEA_ORG:-AndroidCast}}" GITEA_REPO="${GITEA_REPO:-android_cast}" GITEA_MIRROR_INTERVAL="${GITEA_MIRROR_INTERVAL:-10m}" GITEA_PRIVATE="${GITEA_PRIVATE:-false}" -WORK_DIR="${WORK_DIR:-/tmp/gitea-attach-$$}" +WORK_DIR="${WORK_DIR:-}" # First positional argument = mirror URL (convenience). if [ $# -ge 1 ] && [ -n "$1" ]; then @@ -151,22 +151,25 @@ create_repo_via_api() { [ -n "${GITEA_ADMIN_TOKEN:-}" ] || return 1 need_cmd curl _api="$(gitea_api_base)" - _http="$(curl -sS -o /tmp/gitea-create-repo.json -w '%{http_code}' \ + _resp="$(mktemp /tmp/gitea-create-repo-XXXXXX.json)" + _http="$(curl -sS -o "$_resp" -w '%{http_code}' \ -X POST "$_api/admin/users/$GITEA_OWNER/repos" \ -H "Authorization: token $GITEA_ADMIN_TOKEN" \ -H "Content-Type: application/json" \ -d "{\"name\":\"$GITEA_REPO\",\"private\":$GITEA_PRIVATE}")" + _rc=1 if [ "$_http" = "201" ] || [ "$_http" = "200" ]; then log "repo created via API" - return 0 - fi - if [ "$_http" = "409" ]; then + _rc=0 + elif [ "$_http" = "409" ]; then log "repo already exists (API 409)" - return 0 + _rc=0 + else + warn "API create repo HTTP $_http" + cat "$_resp" >&2 2>/dev/null || true fi - warn "API create repo HTTP $_http" - cat /tmp/gitea-create-repo.json >&2 2>/dev/null || true - return 1 + gitea_rm_path "$_resp" + return "$_rc" } gitea_owner_uid() { @@ -211,24 +214,28 @@ EOF ) log "API migrate (pull mirror) uid=$_uid via $_api ..." - _http="$(curl -sS -o /tmp/gitea-migrate.json -w '%{http_code}' \ + _resp="$(mktemp /tmp/gitea-migrate-XXXXXX.json)" + _http="$(curl -sS -o "$_resp" -w '%{http_code}' \ -X POST "$_api/repos/migrate" \ -H "Authorization: token $_token" \ -H "Content-Type: application/json" \ -d "$_payload")" + _rc=1 if [ "$_http" = "201" ] || [ "$_http" = "200" ]; then log "mirror created via API (HTTP $_http)" - cat /tmp/gitea-migrate.json + cat "$_resp" printf '\n' - return 0 + _rc=0 + else + if [ "$_http" = "409" ] || [ "$_http" = "422" ]; then + warn "migrate rejected HTTP $_http — repo may already exist; use mirror settings or delete repo first" + fi + warn "API migrate failed HTTP $_http:" + cat "$_resp" >&2 fi - if [ "$_http" = "409" ] || [ "$_http" = "422" ]; then - warn "migrate rejected HTTP $_http — repo may already exist; use mirror settings or delete repo first" - fi - warn "API migrate failed HTTP $_http:" - cat /tmp/gitea-migrate.json >&2 - return 1 + gitea_rm_path "$_resp" + return "$_rc" } ensure_gitea_repo_exists() { @@ -237,27 +244,43 @@ ensure_gitea_repo_exists() { mirror_via_git() { need_cmd git - mkdir -p "$WORK_DIR" + gitea_prune_stale_tmp + if [ -n "$WORK_DIR" ]; then + mkdir -p "$WORK_DIR" + else + WORK_DIR="$(gitea_make_work_dir gitea-attach)" + fi + trap 'gitea_rm_path "$WORK_DIR"' EXIT INT HUP TERM + _mirror="$WORK_DIR/${GITEA_REPO}.mirror.git" + _rc=1 log "git clone --mirror $GITEA_MIRROR_URL" - rm -rf "$_mirror" - git clone --mirror "$GITEA_MIRROR_URL" "$_mirror" + gitea_rm_path "$_mirror" + if ! git clone --mirror "$GITEA_MIRROR_URL" "$_mirror"; then + gitea_rm_path "$WORK_DIR" + return 1 + fi ensure_gitea_repo_exists _push_base="$(gitea_local_git_url)" - _token="$(resolve_push_token)" || die "need credentials: export GITEA_ADMIN_TOKEN=... or run as root with gitea CLI" + if _token="$(resolve_push_token)"; then + _auth_url="${_push_base#http://}" + _push_url="http://${_token}@${_auth_url}" + log "git push --mirror -> $_push_base (localhost, no subpath)" + if git -C "$_mirror" push --mirror "$_push_url"; then + _rc=0 + log "one-time mirror push complete" + fi + else + die "need credentials: export GITEA_ADMIN_TOKEN=... or run as root with gitea CLI" + fi - _auth_url="${_push_base#http://}" - _push_url="http://${_token}@${_auth_url}" - - log "git push --mirror -> $_push_base (localhost, no subpath)" - git -C "$_mirror" push --mirror "$_push_url" - - log "one-time mirror push complete" warn "for automatic updates, re-run with API migrate or enable pull mirror in Gitea UI" - rm -rf "$WORK_DIR" + gitea_rm_path "$WORK_DIR" + trap - EXIT INT HUP TERM + return "$_rc" } enable_mirror_sync_cron_hint() { diff --git a/examples/crash_reporter/backend/scripts/gitea/gitea_cleanup_tmp.sh b/examples/crash_reporter/backend/scripts/gitea/gitea_cleanup_tmp.sh new file mode 100755 index 0000000..e387c95 --- /dev/null +++ b/examples/crash_reporter/backend/scripts/gitea/gitea_cleanup_tmp.sh @@ -0,0 +1,22 @@ +#!/bin/ash +# +# Remove stale gitea script temp dirs under /tmp (mirror clones, work dirs, json). +# Safe to run anytime; does not touch running jobs younger than GITEA_TMP_MAX_AGE_MIN. +# +# Usage: +# sudo ./gitea_cleanup_tmp.sh +# GITEA_TMP_MAX_AGE_MIN=0 sudo ./gitea_cleanup_tmp.sh # aggressive (all ages) +# +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=gitea_mirror_lib.sh +. "$SCRIPT_DIR/gitea_mirror_lib.sh" + +_before="$(find /tmp -maxdepth 1 \( -name 'gitea-migrate-*' -o -name 'gitea-submodules-*' \ + -o -name 'gitea-attach-*' -o -name 'gitea-mirror-*' -o -name 'gitea-*.json' \) 2>/dev/null | wc -l | tr -d ' ')" +gitea_prune_stale_tmp +_after="$(find /tmp -maxdepth 1 \( -name 'gitea-migrate-*' -o -name 'gitea-submodules-*' \ + -o -name 'gitea-attach-*' -o -name 'gitea-mirror-*' -o -name 'gitea-*.json' \) 2>/dev/null | wc -l | tr -d ' ')" +printf '[gitea-cleanup] max_age_min=%s before=%s after=%s\n' \ + "${GITEA_TMP_MAX_AGE_MIN:-60}" "$_before" "$_after" diff --git a/examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh b/examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh index 98b456c..67df5d3 100755 --- a/examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh +++ b/examples/crash_reporter/backend/scripts/gitea/gitea_migrate_project.sh @@ -36,7 +36,7 @@ GITEA_SKIP_MAIN="${GITEA_SKIP_MAIN:-0}" GITEA_SKIP_SUBMODULES="${GITEA_SKIP_SUBMODULES:-0}" GITEA_SKIP_ORG_SETUP="${GITEA_SKIP_ORG_SETUP:-0}" GITEA_SUBMODULES_CONF="${GITEA_SUBMODULES_CONF:-$SCRIPT_DIR/submodules.defaults.conf}" -WORK_DIR="${WORK_DIR:-/tmp/gitea-migrate-$$}" +WORK_DIR="${WORK_DIR:-}" log() { printf '[gitea-migrate] %s\n' "$*"; } warn() { printf '[gitea-migrate] WARN: %s\n' "$*" >&2; } @@ -257,8 +257,13 @@ if command -v rc-service >/dev/null 2>&1; then rc-service gitea status 2>/dev/null | grep -qi started || rc-service gitea start 2>/dev/null || true fi -mkdir -p "$WORK_DIR" -trap 'rm -rf "$WORK_DIR"' EXIT INT HUP TERM +gitea_prune_stale_tmp +if [ -n "$WORK_DIR" ]; then + mkdir -p "$WORK_DIR" +else + WORK_DIR="$(gitea_make_work_dir gitea-migrate)" +fi +trap 'gitea_rm_path "$WORK_DIR"' EXIT INT HUP TERM log "ini=$GITEA_APP_INI owner=$GITEA_OWNER ref=$GITEA_REF" diff --git a/examples/crash_reporter/backend/scripts/gitea/gitea_mirror_lib.sh b/examples/crash_reporter/backend/scripts/gitea/gitea_mirror_lib.sh index 269588a..5d7071c 100755 --- a/examples/crash_reporter/backend/scripts/gitea/gitea_mirror_lib.sh +++ b/examples/crash_reporter/backend/scripts/gitea/gitea_mirror_lib.sh @@ -21,6 +21,27 @@ gitea_api_base() { printf '%s' "${GITEA_LOCAL_API:-http://127.0.0.1:3000/api/v1}" } +# Remove abandoned gitea-* temp dirs/files (e.g. after kill -9). Default: older than 60 min. +gitea_prune_stale_tmp() { + _min="${GITEA_TMP_MAX_AGE_MIN:-60}" + command -v find >/dev/null 2>&1 || return 0 + find /tmp -maxdepth 1 -type d \ + \( -name 'gitea-migrate-*' -o -name 'gitea-submodules-*' \ + -o -name 'gitea-attach-*' -o -name 'gitea-mirror-*' \) \ + -mmin "+$_min" -exec rm -rf {} + 2>/dev/null || true + find /tmp -maxdepth 1 -type f -name 'gitea-*.json' \ + -mmin "+$_min" -delete 2>/dev/null || true +} + +gitea_make_work_dir() { + _prefix="${1:-gitea-work}" + mktemp -d "/tmp/${_prefix}-XXXXXX" +} + +gitea_rm_path() { + rm -rf "$1" 2>/dev/null || true +} + gitea_resolve_push_token() { if [ -n "${GITEA_ADMIN_TOKEN:-}" ]; then printf '%s' "$GITEA_ADMIN_TOKEN" @@ -103,23 +124,27 @@ gitea_migrate_mirror_api() { } EOF ) - _http="$(curl -sS -o /tmp/gitea-migrate-$$.json -w '%{http_code}' \ + _resp="$(mktemp /tmp/gitea-migrate-XXXXXX.json)" + _http="$(curl -sS -o "$_resp" -w '%{http_code}' \ -X POST "$_api/repos/migrate" \ -H "Authorization: token $_token" \ -H "Content-Type: application/json" \ -d "$_payload")" if [ "$_http" = "201" ] || [ "$_http" = "200" ]; then + gitea_rm_path "$_resp" return 0 fi if [ "$_http" = "409" ] || [ "$_http" = "422" ]; then + gitea_rm_path "$_resp" return 2 fi - if [ -r "/tmp/gitea-migrate-$$.json" ]; then + if [ -r "$_resp" ]; then printf '[gitea-mirror] migrate HTTP %s: ' "$_http" >&2 - head -c 400 "/tmp/gitea-migrate-$$.json" >&2 || true + head -c 400 "$_resp" >&2 || true printf '\n' >&2 fi + gitea_rm_path "$_resp" return 1 } @@ -131,15 +156,25 @@ gitea_mirror_push_git() { command -v git >/dev/null 2>&1 || return 1 _mirror="$_work_dir/${_repo}.mirror.git" - rm -rf "$_mirror" - git clone --mirror "$_clone_url" "$_mirror" || return 1 + _rc=1 + + gitea_rm_path "$_mirror" + if ! git clone --mirror "$_clone_url" "$_mirror"; then + gitea_rm_path "$_mirror" + return 1 + fi + gitea_create_repo_cli "$_owner" "$_repo" "${GITEA_PRIVATE:-false}" || true _push_base="$(gitea_local_git_url_for "$_owner" "$_repo")" - _token="$(gitea_resolve_push_token)" || return 1 - _auth_url="${_push_base#http://}" - git -C "$_mirror" push --mirror "http://${_token}@${_auth_url}" || return 1 - rm -rf "$_mirror" - return 0 + if _token="$(gitea_resolve_push_token)"; then + _auth_url="${_push_base#http://}" + if git -C "$_mirror" push --mirror "http://${_token}@${_auth_url}"; then + _rc=0 + fi + fi + + gitea_rm_path "$_mirror" + return "$_rc" } gitea_repo_exists_api() { @@ -193,11 +228,13 @@ gitea_mirror_set_interval_api() { [ -n "$_token" ] || _token="$(gitea_resolve_push_token)" || return 1 _api="$(gitea_api_base)" _payload="{\"mirror_interval\":\"$_interval\"}" - _http="$(curl -sS -o /tmp/gitea-mirror-interval-$$.json -w '%{http_code}' \ + _resp="$(mktemp /tmp/gitea-mirror-interval-XXXXXX.json)" + _http="$(curl -sS -o "$_resp" -w '%{http_code}' \ -X PATCH "$_api/repos/$_owner/$_repo" \ -H "Authorization: token $_token" \ -H "Content-Type: application/json" \ -d "$_payload")" + gitea_rm_path "$_resp" case "$_http" in 200|204) return 0 ;; *) return 1 ;; @@ -220,7 +257,13 @@ gitea_mirror_pull_repo() { _repo="$2" _owner="${3:-$GITEA_OWNER}" _interval="${4:-${GITEA_MIRROR_INTERVAL:-10m}}" - _work_dir="${5:-/tmp/gitea-mirror-$$}" + _local_work="" + if [ $# -ge 5 ] && [ -n "$5" ]; then + _work_dir="$5" + else + _local_work="$(gitea_make_work_dir gitea-mirror)" + _work_dir="$_local_work" + fi _rc=0 gitea_migrate_mirror_api "$_owner" "$_repo" "$_clone_url" "$_interval" "${GITEA_PRIVATE:-false}" \ @@ -237,5 +280,7 @@ gitea_mirror_pull_repo() { fi mkdir -p "$_work_dir" gitea_mirror_push_git "$_owner" "$_repo" "$_clone_url" "$_work_dir" || _rc=1 + gitea_rm_path "$_work_dir/${_repo}.mirror.git" + [ -n "$_local_work" ] && gitea_rm_path "$_local_work" return "$_rc" } diff --git a/examples/crash_reporter/backend/scripts/gitea/gitea_mirror_submodules.sh b/examples/crash_reporter/backend/scripts/gitea/gitea_mirror_submodules.sh index 54647cd..ef640c0 100755 --- a/examples/crash_reporter/backend/scripts/gitea/gitea_mirror_submodules.sh +++ b/examples/crash_reporter/backend/scripts/gitea/gitea_mirror_submodules.sh @@ -26,7 +26,7 @@ GITEA_OWNER="${GITEA_OWNER:-${GITEA_ORG:-AndroidCast}}" GITEA_MIRROR_INTERVAL="${GITEA_MIRROR_INTERVAL:-24h}" GITEA_PRIVATE="${GITEA_PRIVATE:-false}" GITEA_SUBMODULES_CONF="${GITEA_SUBMODULES_CONF:-$SCRIPT_DIR/submodules.defaults.conf}" -WORK_DIR="${WORK_DIR:-/tmp/gitea-submodules-$$}" +WORK_DIR="${WORK_DIR:-}" SNIPPET_OUT="${SNIPPET_OUT:-$WORK_DIR/gitmodules.gitea-snippet.ini}" log() { printf '[gitea-submodules] %s\n' "$*"; } @@ -159,7 +159,13 @@ if ! gitea_owner_exists "$GITEA_OWNER"; then GITEA_OWNER="$_fb" fi -mkdir -p "$WORK_DIR" +gitea_prune_stale_tmp +if [ -n "$WORK_DIR" ]; then + mkdir -p "$WORK_DIR" +else + WORK_DIR="$(gitea_make_work_dir gitea-submodules)" +fi +trap 'gitea_rm_path "$WORK_DIR"' EXIT INT HUP TERM : > "$WORK_DIR/entries.list" if ! load_submodule_entries "$_gitmodules_arg" | while IFS= read -r _line; do [ -n "$_line" ] || continue diff --git a/examples/crash_reporter/backend/scripts/gitea/gitea_org_lib.sh b/examples/crash_reporter/backend/scripts/gitea/gitea_org_lib.sh index 2193c18..c6e0d73 100644 --- a/examples/crash_reporter/backend/scripts/gitea/gitea_org_lib.sh +++ b/examples/crash_reporter/backend/scripts/gitea/gitea_org_lib.sh @@ -53,19 +53,21 @@ gitea_org_create_api() { } EOF ) - _http="$(curl -sS -o /tmp/gitea-org-create-$$.json -w '%{http_code}' \ + _resp="$(mktemp /tmp/gitea-org-create-XXXXXX.json)" + _http="$(curl -sS -o "$_resp" -w '%{http_code}' \ -X POST "$_api/orgs" \ -H "Authorization: token $_token" \ -H "Content-Type: application/json" \ -d "$_payload")" + _rc=1 case "$_http" in - 201|200) return 0 ;; + 201|200) _rc=0 ;; 409|422) - gitea_org_exists "$_org" && return 0 - return 1 + gitea_org_exists "$_org" && _rc=0 ;; - *) return 1 ;; esac + gitea_rm_path "$_resp" + return "$_rc" } gitea_user_create_if_missing() { @@ -156,19 +158,21 @@ gitea_transfer_repo_api() { _token="$(gitea_api_token)" || return 1 _api="$(gitea_api_base)" _payload="{\"new_owner\":\"$_to\",\"team_ids\":[]}" - _http="$(curl -sS -o /tmp/gitea-transfer-$$.json -w '%{http_code}' \ + _resp="$(mktemp /tmp/gitea-transfer-XXXXXX.json)" + _http="$(curl -sS -o "$_resp" -w '%{http_code}' \ -X POST "$_api/repos/$_from/$_repo/transfer" \ -H "Authorization: token $_token" \ -H "Content-Type: application/json" \ -d "$_payload")" + _rc=1 case "$_http" in - 200|201|202) return 0 ;; + 200|201|202) _rc=0 ;; 409|422) - gitea_repo_exists_api "$_to" "$_repo" && return 0 - return 1 + gitea_repo_exists_api "$_to" "$_repo" && _rc=0 ;; - *) return 1 ;; esac + gitea_rm_path "$_resp" + return "$_rc" } # Move known project mirrors from GITEA_TRANSFER_FROM → org (no-op if absent). diff --git a/examples/network/README.md b/examples/network/README.md new file mode 100644 index 0000000..07f451a --- /dev/null +++ b/examples/network/README.md @@ -0,0 +1,61 @@ +# Edge network probes + +Scripts for the **Debian router** (`134.17.26.161` / `10.7.6.228`) — not FE/BE app hosts. + +## `wan-snapshot.sh` + +Periodic ping comparison: + +| Target | Default | Meaning | +|--------|---------|---------| +| **WAN** | `1.1.1.1` | ISP/upstream leg (outbound from router) | +| **FE** | `10.7.0.10` | LAN leg router → Gentoo FE | +| **BE** | `10.7.16.128` | Optional deeper LAN (disable: `BE_PING_TARGET=none`) | + +### Install (router) + +```bash +sudo cp examples/network/wan-snapshot.sh /usr/local/bin/wan-snapshot.sh +sudo chmod +x /usr/local/bin/wan-snapshot.sh +``` + +### Cron (every 15 minutes) + +```cron +*/15 * * * * root /usr/local/bin/wan-snapshot.sh >>/var/log/wan-quality.cron.log 2>&1 +``` + +Log: `/var/log/wan-quality.log` (override with `WAN_SNAPSHOT_LOG`). + +### Usage + +```bash +# one line appended + printed +sudo /usr/local/bin/wan-snapshot.sh + +# compare tonight vs yesterday +/usr/local/bin/wan-snapshot.sh --report + +# recent samples +/usr/local/bin/wan-snapshot.sh --tail 48 +``` + +Example log line: + +```text +ts=2026-06-04T21:15:01+02:00 wan=1.1.1.1 wan_loss=0% wan_avg_ms=11.2 fe=10.7.0.10 fe_loss=0% fe_avg_ms=0.3 be=10.7.16.128 be_loss=0% be_avg_ms=0.5 +``` + +### Reading results + +- **`wan_avg_ms` high, `fe_avg_ms` low** → ISP/upstream (your “rocketship vs gap” bet). +- **Both high** → router CPU, conntrack, or local link. +- **Ping good, HTTPS/git slow** → DNAT or service layer (not measured here). + +From **outside** LAN, still run occasionally: + +```bash +mtr -rwzc 50 134.17.26.161 +``` + +That catches path issues ping from the router cannot see. diff --git a/examples/network/wan-snapshot.sh b/examples/network/wan-snapshot.sh new file mode 100755 index 0000000..850f8c6 --- /dev/null +++ b/examples/network/wan-snapshot.sh @@ -0,0 +1,203 @@ +#!/bin/sh +# +# Log WAN vs LAN ping quality on the Debian edge router. +# Use to compare "rocketship" evenings vs slow periods (ISP/upstream vs local). +# +# Topology (see docs/INFRA.md): +# world -> 134.17.26.161 (router 10.7.6.228) -> 10.7.0.10 (FE) -> 10.7.0.0/8 (BE, …) +# +# Install on router: +# sudo cp wan-snapshot.sh /usr/local/bin/wan-snapshot.sh +# sudo chmod +x /usr/local/bin/wan-snapshot.sh +# sudo mkdir -p /var/log +# +# Cron (every 15 minutes): +# */15 * * * * root /usr/local/bin/wan-snapshot.sh +# +# Env overrides: +# WAN_SNAPSHOT_LOG=/var/log/wan-quality.log +# WAN_PING_TARGET=1.1.1.1 +# FE_PING_TARGET=10.7.0.10 +# BE_PING_TARGET=10.7.16.128 (set to "none" to skip) +# PING_COUNT=10 +# PING_DEADLINE_S=15 +# +set -eu + +WAN_SNAPSHOT_LOG="${WAN_SNAPSHOT_LOG:-/var/log/wan-quality.log}" +WAN_PING_TARGET="${WAN_PING_TARGET:-1.1.1.1}" +FE_PING_TARGET="${FE_PING_TARGET:-10.7.0.10}" +BE_PING_TARGET="${BE_PING_TARGET-10.7.16.128}" +PING_COUNT="${PING_COUNT:-10}" +PING_DEADLINE_S="${PING_DEADLINE_S:-15}" + +usage() { + sed -n '2,22p' "$0" + printf '\nCommands:\n' + printf ' (default) append one snapshot line to the log\n' + printf ' --report summarize today vs yesterday averages\n' + printf ' --tail [N] print last N log lines (default 24)\n' + printf ' --help this text\n' +} + +need_ping() { + command -v ping >/dev/null 2>&1 || { + echo "wan-snapshot: missing ping" >&2 + exit 1 + } +} + +# ping_summary HOST -> prints: loss_pct avg_ms (or loss_pct=-1 avg_ms=-1 on total failure) +ping_summary() { + _host="$1" + _out="" + _out="$(ping -c "$PING_COUNT" -w "$PING_DEADLINE_S" "$_host" 2>/dev/null)" || true + if [ -z "$_out" ]; then + printf '%s %s' "-1" "-1" + return 0 + fi + _loss="$(printf '%s\n' "$_out" | awk '/packet loss/ { + for (i = 1; i <= NF; i++) if ($i ~ /%$/) { gsub(/%/, "", $i); print $i; exit } + }')" + _avg="$(printf '%s\n' "$_out" | awk -F'[=/ ]' '/min\/avg\/max/ { print $8; exit }')" + [ -n "$_loss" ] || _loss="-1" + [ -n "$_avg" ] || _avg="-1" + printf '%s %s' "$_loss" "$_avg" +} + +snapshot_once() { + need_ping + _ts="$(date -Iseconds 2>/dev/null || date '+%Y-%m-%dT%H:%M:%S%z')" + _wan="$(ping_summary "$WAN_PING_TARGET")" + _wan_loss="${_wan%% *}" + _wan_avg="${_wan#* }" + _fe="$(ping_summary "$FE_PING_TARGET")" + _fe_loss="${_fe%% *}" + _fe_avg="${_fe#* }" + + _line="ts=$_ts wan=$WAN_PING_TARGET wan_loss=${_wan_loss}% wan_avg_ms=${_wan_avg} fe=$FE_PING_TARGET fe_loss=${_fe_loss}% fe_avg_ms=${_fe_avg}" + + if [ -n "$BE_PING_TARGET" ] && [ "$BE_PING_TARGET" != "none" ]; then + _be="$(ping_summary "$BE_PING_TARGET")" + _be_loss="${_be%% *}" + _be_avg="${_be#* }" + _line="$_line be=$BE_PING_TARGET be_loss=${_be_loss}% be_avg_ms=${_be_avg}" + fi + + _dir="$(dirname "$WAN_SNAPSHOT_LOG")" + [ -d "$_dir" ] || mkdir -p "$_dir" 2>/dev/null || true + printf '%s\n' "$_line" >> "$WAN_SNAPSHOT_LOG" + printf '%s\n' "$_line" +} + +# Average wan_avg_ms / fe_avg_ms for lines matching DATE prefix (YYYY-MM-DD). +day_avg() { + _log="$1" + _date="$2" + _field="$3" + [ -r "$_log" ] || return 0 + awk -v d="$_date" -v f="$_field" ' + $0 ~ "^ts=" d { + for (i = 1; i <= NF; i++) { + if ($i ~ "^" f "=") { + split($i, a, "=") + v = a[2] + gsub(/%/, "", v) + if (v + 0 == v && v >= 0) { n++; s += v } + } + } + } + END { + if (n > 0) printf "%.2f", s / n + else print "n/a" + } + ' "$_log" +} + +day_max_loss() { + _log="$1" + _date="$2" + _field="$3" + [ -r "$_log" ] || return 0 + awk -v d="$_date" -v f="$_field" ' + $0 ~ "^ts=" d { + for (i = 1; i <= NF; i++) { + if ($i ~ "^" f "=") { + split($i, a, "=") + v = a[2] + gsub(/%/, "", v) + if (v + 0 == v && v >= 0 && v > max) max = v + } + } + } + END { + if (max == "") print "n/a" + else printf "%.0f", max + } + ' "$_log" +} + +report() { + _log="$WAN_SNAPSHOT_LOG" + _today="$(date '+%Y-%m-%d')" + _yesterday="$(date -d 'yesterday' '+%Y-%m-%d' 2>/dev/null || date -v-1d '+%Y-%m-%d' 2>/dev/null || echo '')" + + if [ ! -r "$_log" ]; then + echo "wan-snapshot: no log at $_log (run default command first)" >&2 + exit 1 + fi + + _samples_today="$(grep -c "^ts=$_today" "$_log" 2>/dev/null || echo 0)" + printf 'log=%s\n' "$_log" + printf 'today=%s samples=%s\n' "$_today" "$_samples_today" + printf ' wan_avg_ms=%s wan_max_loss%%=%s\n' \ + "$(day_avg "$_log" "$_today" "wan_avg_ms")" \ + "$(day_max_loss "$_log" "$_today" "wan_loss")" + printf ' fe_avg_ms=%s fe_max_loss%%=%s\n' \ + "$(day_avg "$_log" "$_today" "fe_avg_ms")" \ + "$(day_max_loss "$_log" "$_today" "fe_loss")" + + if [ -n "$_yesterday" ] && grep -q "^ts=$_yesterday" "$_log" 2>/dev/null; then + _samples_y="$(grep -c "^ts=$_yesterday" "$_log" 2>/dev/null || echo 0)" + printf 'yesterday=%s samples=%s\n' "$_yesterday" "$_samples_y" + printf ' wan_avg_ms=%s wan_max_loss%%=%s\n' \ + "$(day_avg "$_log" "$_yesterday" "wan_avg_ms")" \ + "$(day_max_loss "$_log" "$_yesterday" "wan_loss")" + printf ' fe_avg_ms=%s fe_max_loss%%=%s\n' \ + "$(day_avg "$_log" "$_yesterday" "fe_avg_ms")" \ + "$(day_max_loss "$_log" "$_yesterday" "fe_loss")" + fi + + printf '\nInterpretation:\n' + printf ' wan bad + fe good -> likely ISP/upstream (world -> 134.17.26.161)\n' + printf ' wan + fe bad -> router or LAN (10.7.6.228 -> 10.7.0.10)\n' + printf ' wan + fe good, apps slow -> DNAT/nginx above ping path\n' +} + +tail_log() { + _n="${1:-24}" + [ -r "$WAN_SNAPSHOT_LOG" ] || { + echo "wan-snapshot: no log at $WAN_SNAPSHOT_LOG" >&2 + exit 1 + } + tail -n "$_n" "$WAN_SNAPSHOT_LOG" +} + +case "${1:-}" in + -h|--help|help) + usage + ;; + --report|report) + report + ;; + --tail|tail) + tail_log "${2:-24}" + ;; + "") + snapshot_once + ;; + *) + echo "wan-snapshot: unknown command: $1 (try --help)" >&2 + exit 1 + ;; +esac