diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4f421b9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +storage/* +!storage/.gitkeep +run/ +*.log +config/local.env +config/yt-dlp.auth.env +.stream_pipe +.current_source +.current_index +.intro_shown diff --git a/README.md b/README.md index 1464930..f318270 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,35 @@ # ac-ms-media-transcode -Future VOD/transcode. Remote: `git://f0xx.org/ac/ac-ms-media-transcode` +V.2x lab restream / transcode worker (F2). Shell pipeline: **encode once → HLS + optional RTMP fanout**. + +Remote: `git://f0xx.org/ac/ac-ms-media-transcode` + +## Layout + +| Path | Role | +|------|------| +| `lib/env.sh` | **Single config source** — all variables, overload via `export` or `config/local.env` | +| `assets/pirate_drift.png` | Default logo watermark (committed) | +| `bin/` | encode-once, fanout-copy, run, probe, … | +| `config/sources.list` | Lab source URLs | +| `config/sinks.list` | RTMP publish targets (fanout stage) | + +## Lab cluster + +Composed BE `config.php` (via `compose-lab-backend.sh`): + +- `media_pipelines.enabled` = **true** +- `script_root` = `/var/www/ac/workspace/ac-ms-media-transcode` +- `logo_path` = `…/assets/pirate_drift.png` +- HLS nginx alias = `/app/androidcast_project/media/hls/` → `/var/www/ac/media-pipelines/hls/` + +Console UI: `?view=media_pipelines` (global admin / `media_pipelines_*` grants). + +## Local override + +```bash +cp config/local.env.example config/local.env +./bin/run.sh 1 +``` + +All tunables live in `lib/env.sh` using `export VAR="${VAR:-default}"`. diff --git a/assets/README.txt b/assets/README.txt new file mode 100644 index 0000000..fc0d87e --- /dev/null +++ b/assets/README.txt @@ -0,0 +1,3 @@ +Default watermark: **pirate_drift.png** (committed in this repo). + +Override: `export LOGO_PATH=/other/path.png` or set in `config/local.env`. diff --git a/assets/pirate_drift.png b/assets/pirate_drift.png new file mode 100644 index 0000000..1d12e18 Binary files /dev/null and b/assets/pirate_drift.png differ diff --git a/bin/encode-once.sh b/bin/encode-once.sh new file mode 100755 index 0000000..6569404 --- /dev/null +++ b/bin/encode-once.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Single encode: ingest → filter (logo/delogo) → tee → local file + named pipe (+ optional HLS). +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck disable=SC1091 +source "${ROOT}/lib/common.sh" +# shellcheck disable=SC1091 +source "${ROOT}/lib/source.sh" +# shellcheck disable=SC1091 +source "${ROOT}/lib/filters.sh" +# shellcheck disable=SC1091 +source "${ROOT}/lib/outputs.sh" + +resolve_source "${1:-}" +require_cmd "${FFMPEG_BIN}" +require_logo +ensure_storage +ensure_pipe + +compute_layout() { + local idx=0 + VIDEO_INPUT="${idx}:v" + if [[ "${USE_YTDLP}" == "1" ]]; then + idx=0 + else + idx=0 + fi + if [[ ${#AUDIO_INPUT_ARGS[@]} -gt 0 ]]; then + AUDIO_MAP_INPUT=$((idx + 1)) + LOGO_INPUT=$((idx + 2)) + else + AUDIO_MAP_INPUT="${idx}" + LOGO_INPUT=$((idx + 1)) + fi +} + +compute_layout +RECORD_PATH="$(next_record_path)" +LOG_FILE="${STORAGE_DIR}/encode.log" +build_filter_complex "${VIDEO_INPUT}" "${LOGO_INPUT}:v" +TEE_SPEC="$(build_encode_tee_spec "${RECORD_PATH}" "${PIPE_PATH}")" +MAP_V="$(map_video_out)" +MAP_A="$(map_audio_out)" + +run_ffmpeg() { + "${FFMPEG_BIN}" -hide_banner -loglevel info \ + -rtbufsize 256M -probesize 100M -analyzeduration 250M \ + "$@" \ + -i "${LOGO_PATH}" \ + "${FFMPEG_TS_FLAGS[@]}" \ + -filter_complex "${FILTER_COMPLEX}" \ + -map "${MAP_V}" -map "${MAP_A}" \ + -threads "${THREADS}" -framerate "${FPS}" -r "${FPS}" \ + -s "${RES}" ${VCODEC} ${PRESET} ${BITRATE} ${ACODEC} -bufsize "${BUFSIZE}" \ + -f tee "${TEE_SPEC}" >> "${LOG_FILE}" 2>&1 +} + +cleanup() { + remove_pipe +} +trap cleanup EXIT + +log "recording → ${RECORD_PATH}" +log "pipe → ${PIPE_PATH}" +[[ -n "${HLS_OUTPUT_DIR}" ]] && log "HLS → ${HLS_OUTPUT_DIR}" + +if [[ "${USE_YTDLP}" == "1" ]]; then + require_cmd "${YTDLP_BIN}" + mapfile -d '' -t YTDLP_CMD < <(build_ytdlp_pipeline) + "${YTDLP_CMD[@]}" 2>>"${LOG_FILE}" | \ + run_ffmpeg -thread_queue_size "${THREAD_QUEUE_SIZE}" -i pipe:0 \ + "${AUDIO_INPUT_ARGS[@]}" +else + read_rate_flag + run_ffmpeg \ + "${RE_FLAG[@]}" \ + -thread_queue_size "${THREAD_QUEUE_SIZE}" \ + -reconnect_on_network_error 1 -reconnect_streamed 1 -reconnect_delay_max 5 \ + -i "${SOURCE_URL}" \ + "${AUDIO_INPUT_ARGS[@]}" +fi diff --git a/bin/encode-only.sh b/bin/encode-only.sh new file mode 100755 index 0000000..785b61f --- /dev/null +++ b/bin/encode-only.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Encode once → local recording (+ optional HLS). No named-pipe fanout stage. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck disable=SC1091 +source "${ROOT}/lib/common.sh" +# shellcheck disable=SC1091 +source "${ROOT}/lib/source.sh" +# shellcheck disable=SC1091 +source "${ROOT}/lib/filters.sh" +# shellcheck disable=SC1091 +source "${ROOT}/lib/outputs.sh" + +resolve_source "${1:-}" +require_cmd "${FFMPEG_BIN}" +require_logo +ensure_storage + +compute_layout() { + VIDEO_INPUT="0:v" + if [[ ${#AUDIO_INPUT_ARGS[@]} -gt 0 ]]; then + AUDIO_MAP_INPUT=1 + LOGO_INPUT=2 + else + AUDIO_MAP_INPUT=0 + LOGO_INPUT=1 + fi +} + +compute_layout +RECORD_PATH="$(next_record_path)" +LOG_FILE="${STORAGE_DIR}/encode.log" +build_filter_complex "${VIDEO_INPUT}" "${LOGO_INPUT}:v" +MAP_V="$(map_video_out)" +MAP_A="$(map_audio_out)" + +build_record_tee() { + local record="$1" + local parts=() + parts+=("$(file_tee_entry "${record}")") + if [[ -n "${HLS_OUTPUT_DIR}" ]]; then + parts+=("$(hls_tee_entry "${HLS_OUTPUT_DIR}")") + fi + local IFS='|' + echo "${parts[*]}" +} + +TEE_SPEC="$(build_record_tee "${RECORD_PATH}")" + +run_ffmpeg() { + "${FFMPEG_BIN}" -hide_banner -loglevel info \ + -rtbufsize 256M -probesize 100M -analyzeduration 250M \ + "$@" \ + -i "${LOGO_PATH}" \ + "${FFMPEG_TS_FLAGS[@]}" \ + -filter_complex "${FILTER_COMPLEX}" \ + -map "${MAP_V}" -map "${MAP_A}" \ + -threads "${THREADS}" -framerate "${FPS}" -r "${FPS}" \ + -s "${RES}" ${VCODEC} ${PRESET} ${BITRATE} ${ACODEC} -bufsize "${BUFSIZE}" \ + -f tee "${TEE_SPEC}" >> "${LOG_FILE}" 2>&1 +} + +log "encode-only → ${RECORD_PATH}" +[[ -n "${HLS_OUTPUT_DIR}" ]] && log "HLS → ${HLS_OUTPUT_DIR}" + +if [[ "${USE_YTDLP}" == "1" ]]; then + require_cmd "${YTDLP_BIN}" + mapfile -d '' -t YTDLP_CMD < <(build_ytdlp_pipeline) + "${YTDLP_CMD[@]}" 2>>"${LOG_FILE}" | \ + run_ffmpeg -thread_queue_size "${THREAD_QUEUE_SIZE}" -i pipe:0 \ + "${AUDIO_INPUT_ARGS[@]}" +else + read_rate_flag + run_ffmpeg \ + "${RE_FLAG[@]}" \ + -thread_queue_size "${THREAD_QUEUE_SIZE}" \ + -reconnect_on_network_error 1 -reconnect_streamed 1 -reconnect_delay_max 5 \ + -i "${SOURCE_URL}" \ + "${AUDIO_INPUT_ARGS[@]}" +fi diff --git a/bin/fanout-copy.sh b/bin/fanout-copy.sh new file mode 100755 index 0000000..22a6182 --- /dev/null +++ b/bin/fanout-copy.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Fan-out: read pre-encoded FLV from pipe → RTMP sinks (stream copy, no re-encode). +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck disable=SC1091 +source "${ROOT}/lib/common.sh" +# shellcheck disable=SC1091 +source "${ROOT}/lib/outputs.sh" + +require_cmd "${FFMPEG_BIN}" +ensure_storage +ensure_pipe + +TEE_SPEC="$(build_fanout_tee_spec)" +LOG_FILE="${STORAGE_DIR}/fanout.log" +MAX_RETRIES="${FANOUT_RETRIES:-5}" +attempt=0 + +log "fanout → ${#SINKS[@]} sink(s)" + +while [[ "${attempt}" -le "${MAX_RETRIES}" ]]; do + attempt=$((attempt + 1)) + log "fanout attempt ${attempt}/${MAX_RETRIES}" + if "${FFMPEG_BIN}" -hide_banner -loglevel info \ + -thread_queue_size "${THREAD_QUEUE_SIZE}" \ + -i "${PIPE_PATH}" \ + -map 0 \ + -c copy \ + -f tee "${TEE_SPEC}" >> "${LOG_FILE}" 2>&1; then + log "fanout finished normally" + break + fi + warn "fanout ffmpeg exited — retrying after pipe flush" + sleep 1 +done diff --git a/bin/install-nginx-hls.sh b/bin/install-nginx-hls.sh new file mode 100755 index 0000000..4a0ea28 --- /dev/null +++ b/bin/install-nginx-hls.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Lab helper: print nginx fragment with HLS_ROOT substituted. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +HLS_ROOT="${1:-${ROOT}/storage/hls}" + +sed "s|@HLS_ROOT@|${HLS_ROOT}|g" "${ROOT}/nginx/hls-location.conf" diff --git a/bin/probe-source.sh b/bin/probe-source.sh new file mode 100755 index 0000000..983e0db --- /dev/null +++ b/bin/probe-source.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# ffprobe → normalized source profile (draft SPEC-1 preview). +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck disable=SC1091 +source "${ROOT}/lib/common.sh" +# shellcheck disable=SC1091 +source "${ROOT}/lib/source.sh" + +resolve_source "${1:-}" +require_cmd "${FFPROBE_BIN}" + +probe_target="${SOURCE_URL}" +if [[ "${USE_YTDLP}" == "1" ]]; then + warn "page URL — ffprobe may fail; use a direct m3u8/mp4 URL or run encode-once" +fi + +log "probing ${probe_target}" +"${FFPROBE_BIN}" -hide_banner -loglevel error \ + -show_format -show_streams -print_format json \ + "${probe_target}" | python3 -c ' +import json, sys +data = json.load(sys.stdin) +fmt = data.get("format", {}) +streams = data.get("streams", []) +v = next((s for s in streams if s.get("codec_type") == "video"), None) +a = next((s for s in streams if s.get("codec_type") == "audio"), None) +out = { + "duration": fmt.get("duration"), + "bitrate": fmt.get("bit_rate"), + "format": fmt.get("format_name"), + "video": None, + "audio": None, +} +if v: + out["video"] = { + "codec": v.get("codec_name"), + "width": v.get("width"), + "height": v.get("height"), + "fps": v.get("avg_frame_rate"), + "profile": v.get("profile"), + } +if a: + out["audio"] = { + "codec": a.get("codec_name"), + "channels": a.get("channels"), + "sample_rate": a.get("sample_rate"), + } +print(json.dumps(out, indent=2)) +' diff --git a/bin/run-encode-only.sh b/bin/run-encode-only.sh new file mode 100755 index 0000000..de4edf1 --- /dev/null +++ b/bin/run-encode-only.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck disable=SC1091 +source "${ROOT}/lib/common.sh" + +SRC_ARG="${1:-1}" +PID_ENCODE="${RUN_DIR}/encode.pid" +LOG_RUN="${STORAGE_DIR}/run-encode-only.log" + +ensure_storage +mkdir -p "${RUN_DIR}" + +if [[ -f "${PID_ENCODE}" ]] && kill -0 "$(cat "${PID_ENCODE}")" 2>/dev/null; then + die "encode-only already running (pid $(cat "${PID_ENCODE}")). use bin/stop.sh first" +fi + +chmod +x "${ROOT}/bin/"*.sh 2>/dev/null || true +log "starting encode-only source=${SRC_ARG}" +nohup "${ROOT}/bin/encode-only.sh" "${SRC_ARG}" >> "${LOG_RUN}" 2>&1 & +echo $! > "${PID_ENCODE}" +log "encode pid $(cat "${PID_ENCODE}")" diff --git a/bin/run.sh b/bin/run.sh new file mode 100755 index 0000000..000fe1b --- /dev/null +++ b/bin/run.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Start encode-once + fanout-copy (two-stage pipeline from v0). +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck disable=SC1091 +source "${ROOT}/lib/common.sh" + +SRC_ARG="${1:-1}" +PID_ENCODE="${RUN_DIR}/encode.pid" +PID_FANOUT="${RUN_DIR}/fanout.pid" +LOG_RUN="${STORAGE_DIR}/run.log" + +ensure_storage +mkdir -p "${RUN_DIR}" + +if [[ -f "${PID_ENCODE}" ]] && kill -0 "$(cat "${PID_ENCODE}")" 2>/dev/null; then + die "pipeline already running (encode pid $(cat "${PID_ENCODE}")). use bin/stop.sh first" +fi + +chmod +x "${ROOT}/bin/"*.sh 2>/dev/null || true + +# shellcheck disable=SC1091 +source "${ROOT}/lib/outputs.sh" +read_sinks + +log "starting pipeline source=${SRC_ARG}" +nohup "${ROOT}/bin/encode-once.sh" "${SRC_ARG}" >> "${LOG_RUN}" 2>&1 & +echo $! > "${PID_ENCODE}" +sleep 1 + +if [[ ${#SINKS[@]} -gt 0 ]]; then + nohup "${ROOT}/bin/fanout-copy.sh" >> "${LOG_RUN}" 2>&1 & + echo $! > "${PID_FANOUT}" + log "fanout pid $(cat "${PID_FANOUT}")" +else + log "no RTMP sinks configured — fanout stage skipped" + rm -f "${PID_FANOUT}" 2>/dev/null || true +fi + +log "encode pid $(cat "${PID_ENCODE}")" +log "logs: ${LOG_RUN}, ${STORAGE_DIR}/encode.log, ${STORAGE_DIR}/fanout.log" diff --git a/bin/stop.sh b/bin/stop.sh new file mode 100755 index 0000000..b910a0f --- /dev/null +++ b/bin/stop.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck disable=SC1091 +source "${ROOT}/lib/common.sh" + +stop_pidfile() { + local pf="$1" + [[ -f "${pf}" ]] || return 0 + local pid + pid="$(cat "${pf}")" + if kill -0 "${pid}" 2>/dev/null; then + kill "${pid}" 2>/dev/null || true + sleep 1 + kill -9 "${pid}" 2>/dev/null || true + fi + rm -f "${pf}" +} + +stop_pidfile "${RUN_DIR}/fanout.pid" +stop_pidfile "${RUN_DIR}/encode.pid" +pkill -f "${ROOT}/bin/fanout-copy.sh" 2>/dev/null || true +pkill -f "${ROOT}/bin/encode-once.sh" 2>/dev/null || true +remove_pipe +rm -f "${ROOT}/.intro_shown" "${ROOT}/.current_source" 2>/dev/null || true +log "pipeline stopped" diff --git a/bin/switch-source.sh b/bin/switch-source.sh new file mode 100755 index 0000000..d27e3ad --- /dev/null +++ b/bin/switch-source.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Stop pipeline, optionally rotate source index, restart (v0 switch_quality.sh cleanup). +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck disable=SC1091 +source "${ROOT}/lib/common.sh" + +VARIANT="${1:-}" +SRC="${SRC:-}" + +if [[ "${VARIANT}" == "stop" ]]; then + exec "${ROOT}/bin/stop.sh" +fi + +case "${VARIANT}" in + blur|blurred) export FILTER_PROFILE=blurred ;; + nologo) export FILTER_PROFILE=nologo ;; + default|'') export FILTER_PROFILE=default ;; + encode-only|record) + export RUN_ENCODE_ONLY=1 + ;; + [0-9]*) + SRC="${VARIANT}" + ;; +esac + +if [[ -z "${SRC}" && -f "${ROOT}/.current_source" ]]; then + # rotate index in sources.list + if [[ -f "${SOURCES_LIST}" ]]; then + count="$(grep -vc '^[[:space:]]*#' "${SOURCES_LIST}" || true)" + cur=1 + [[ -f "${ROOT}/.current_index" ]] && cur="$(cat "${ROOT}/.current_index")" + cur=$((cur + 1)) + [[ "${count}" -gt 0 && "${cur}" -gt "${count}" ]] && cur=1 + echo "${cur}" > "${ROOT}/.current_index" + SRC="${cur}" + fi +fi + +SRC="${SRC:-1}" + +"${ROOT}/bin/stop.sh" +sleep 1 +log "switch → source index ${SRC} profile ${FILTER_PROFILE}" +if [[ -n "${RUN_ENCODE_ONLY:-}" || "${VARIANT}" == "encode-only" || "${VARIANT}" == "record" ]]; then + exec "${ROOT}/bin/run-encode-only.sh" "${SRC}" +fi + +exec "${ROOT}/bin/run.sh" "${SRC}" diff --git a/config/local.env.example b/config/local.env.example new file mode 100644 index 0000000..ed569c9 --- /dev/null +++ b/config/local.env.example @@ -0,0 +1,6 @@ +# Copy to config/local.env — overrides lib/env.sh defaults (sourced last among repo config). + +# HLS on disk for nginx / web player: +# HLS_OUTPUT_DIR="${ROOT}/storage/hls/live" + +# Lab RTMP sinks: edit config/sinks.list or per-pipeline work dir sinks.list diff --git a/config/sinks.example.list b/config/sinks.example.list new file mode 100644 index 0000000..f46e3b5 --- /dev/null +++ b/config/sinks.example.list @@ -0,0 +1,5 @@ +# RTMP (or other ffmpeg tee URL) destinations — fanout stage copies without re-encode. +# Lines starting with # are ignored. +# +# rtmp://127.0.0.1/live/stream_key +# rtmp://a.rtmp.youtube.com/live2/STREAM_KEY diff --git a/config/sinks.list b/config/sinks.list new file mode 100644 index 0000000..d119afc --- /dev/null +++ b/config/sinks.list @@ -0,0 +1,14 @@ +# Migrated from streaming_v0 (sanitized — replace STREAM_KEY / host for your lab). +# Active sinks for fanout-copy stage (stream copy, no re-encode). +# +# Local nginx-rtmp or MediaMTX lab ingest: +# rtmp://127.0.0.1/live/dev +# +# YouTube Live (lab only — paste stream key from YouTube Studio): +# rtmp://a.rtmp.youtube.com/live2/STREAM_KEY_HERE +# +# Twitch: +# rtmp://live.twitch.tv/app/STREAM_KEY_HERE +# +# Previous v0 hopto relay (example — do not use without your own endpoint): +# rtmp://your-relay.example.org/live/stream diff --git a/config/sources.example.list b/config/sources.example.list new file mode 100644 index 0000000..803b212 --- /dev/null +++ b/config/sources.example.list @@ -0,0 +1,6 @@ +# One source per line: URL, file path, or page URL (yt-dlp when YTDLP_ENABLED=1). +# Optional second column: yt-dlp format selector (e.g. bv*+ba/b). +# +# https://example.com/live/master.m3u8 +# /path/to/local/file.mkv +# https://www.youtube.com/watch?v=EXAMPLE bv*+ba/b diff --git a/config/sources.list b/config/sources.list new file mode 100644 index 0000000..ea65149 --- /dev/null +++ b/config/sources.list @@ -0,0 +1,13 @@ +# Migrated from streaming_v0 (sanitized — no expiring CDN tokens or credentials). +# Format: URL [yt-dlp format selector] +# +# Page URLs (YouTube, etc.) — require YTDLP_ENABLED=1 in defaults.env or local.env +https://www.youtube.com/watch?v=WQ4mwufOjEI +https://www.youtube.com/watch?v=xaiQUHz8re4 +https://www.youtube.com/watch?v=Z-QVb-3D6zA +https://www.youtube.com/watch?v=Trwer1xXYtY +https://www.youtube.com/watch?v=n2CxmPwqq8w +# +# Direct HLS / file examples (uncomment for tests) +# https://example.com/live/master.m3u8 +# /path/to/local/recording.mkv diff --git a/config/yt-dlp.auth.example.env b/config/yt-dlp.auth.example.env new file mode 100644 index 0000000..ff8e39b --- /dev/null +++ b/config/yt-dlp.auth.example.env @@ -0,0 +1,7 @@ +# Copy to config/yt-dlp.auth.env — never commit real credentials. +# Sourced only when yt-dlp needs authenticated pages (lab / non-prod). + +# YTDLP_USERNAME= +# YTDLP_PASSWORD= +# YTDLP_NETRC=1 +# YTDLP_EXTRA_ARGS=--referer https://example.com/ diff --git a/lib/common.sh b/lib/common.sh new file mode 100644 index 0000000..aff65d9 --- /dev/null +++ b/lib/common.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# Shared bootstrap: ROOT, env defaults, encoding presets, pipe helpers. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +export ROOT + +# shellcheck disable=SC1091 +source "${ROOT}/lib/logging.sh" +# shellcheck disable=SC1091 +source "${ROOT}/lib/env.sh" + +apply_encoding_preset() { + case "${OUTPUT_PRESET}" in + 720p) + WIDTH=1280 + HEIGHT=720 + FASTNESS=veryfast + BV=2500k + BUFSIZE=5000k + ;; + 1080p|*) + WIDTH=1920 + HEIGHT=1080 + FASTNESS=faster + BV=5000k + BUFSIZE=10000k + ;; + esac + export WIDTH HEIGHT FASTNESS BV BUFSIZE + RES="${WIDTH}x${HEIGHT}" + PROFILE=high + KEYFRAMES_INTL="${FPS}" + GOP_INTL=$((KEYFRAMES_INTL * 2)) + PIX_FMT="-pix_fmt yuv420p" + PRESET="-preset ${FASTNESS} -tune zerolatency -profile:v ${PROFILE} ${PIX_FMT} -x264-params keyint=${KEYFRAMES_INTL}:min-keyint=${KEYFRAMES_INTL} -g ${GOP_INTL} -flags +global_header" + BITRATE="-b:v ${BV} -maxrate ${BV} -minrate ${BV}" + VCODEC="-c:v libx264" + ACODEC="-c:a aac -b:a 128k" + export RES PROFILE PRESET BITRATE VCODEC ACODEC +} + +apply_encoding_preset + +FFMPEG_TS_FLAGS=( + -fflags +genpts+igndts + -avoid_negative_ts make_zero + -sn + -movflags +faststart +) + +read_rate_flag() { + if [[ "${NEAR_REALTIME}" == "1" ]]; then + RE_FLAG=(-re) + else + RE_FLAG=() + fi +} + +ensure_storage() { + mkdir -p "${STORAGE_DIR}" "${RUN_DIR}" +} + +ensure_pipe() { + if [[ ! -p "${PIPE_PATH}" ]]; then + mkfifo "${PIPE_PATH}" + if command -v perl >/dev/null 2>&1; then + perl -MFcntl -e 'fcntl(STDIN, 1031, shift) or die $!' "${PIPE_FIFO_SIZE}" <> "${PIPE_PATH}" || true + fi + fi + exec 7<> "${PIPE_PATH}" +} + +remove_pipe() { + exec 7>&- 7<&- 2>/dev/null || true + rm -f "${PIPE_PATH}" 2>/dev/null || true +} + +next_record_path() { + local counter=0 name + ensure_storage + while true; do + name="$(printf "video%04d.%s" "${counter}" "${RECORD_FORMAT}")" + if [[ ! -f "${STORAGE_DIR}/${name}" ]]; then + echo "${STORAGE_DIR}/${name}" + return + fi + counter=$((counter + 1)) + done +} + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || die "required command not found: $1" +} + +require_logo() { + [[ -f "${LOGO_PATH}" ]] || die "logo not found at ${LOGO_PATH} (expected ${ROOT}/assets/pirate_drift.png)" +} diff --git a/lib/env.sh b/lib/env.sh new file mode 100644 index 0000000..ca2a900 --- /dev/null +++ b/lib/env.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Single source of truth for pipeline variables. +# Override: export VAR=value before running, or set in config/local.env (sourced last). +# +# Requires ROOT (set by lib/common.sh before this file is sourced). + +if [[ -z "${ROOT:-}" ]]; then + ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + export ROOT +fi + +# --- binaries --- +export FFMPEG_BIN="${FFMPEG_BIN:-ffmpeg}" +export FFPROBE_BIN="${FFPROBE_BIN:-ffprobe}" +export YTDLP_BIN="${YTDLP_BIN:-yt-dlp}" + +# --- paths (logo default: assets/pirate_drift.png in repo root) --- +export LOGO_PATH="${LOGO_PATH:-${ROOT}/assets/pirate_drift.png}" +export PIPE_NAME="${PIPE_NAME:-.stream_pipe}" +export PIPE_PATH="${PIPE_PATH:-${ROOT}/${PIPE_NAME}}" +export PIPE_FIFO_SIZE="${PIPE_FIFO_SIZE:-1048576}" +export STORAGE_DIR="${STORAGE_DIR:-${ROOT}/storage}" +export RUN_DIR="${RUN_DIR:-${ROOT}/run}" +export SOURCES_LIST="${SOURCES_LIST:-${ROOT}/config/sources.list}" +export SINKS_LIST="${SINKS_LIST:-${ROOT}/config/sinks.list}" +export HLS_OUTPUT_DIR="${HLS_OUTPUT_DIR:-}" + +# --- encode quality --- +export OUTPUT_PRESET="${OUTPUT_PRESET:-1080p}" +export FILTER_PROFILE="${FILTER_PROFILE:-default}" +export FPS="${FPS:-30}" +export THREADS="${THREADS:-8}" +export THREAD_QUEUE_SIZE="${THREAD_QUEUE_SIZE:-512}" +export RECORD_FORMAT="${RECORD_FORMAT:-mkv}" + +# --- ingest behaviour --- +export YTDLP_ENABLED="${YTDLP_ENABLED:-1}" +export NEAR_REALTIME="${NEAR_REALTIME:-1}" +export FANOUT_RETRIES="${FANOUT_RETRIES:-5}" + +# optional delogo rectangle (source coords): x:y:w:h +export DELOGO="${DELOGO:-}" + +# separate audio when video track has no audio +export AUDIO_SOURCE="${AUDIO_SOURCE:-}" + +# --- optional file overrides (highest priority among defaults) --- +_mp_load_env_file() { + local f="$1" + if [[ -f "${f}" ]]; then + set -a + # shellcheck disable=SC1090 + source "${f}" + set +a + fi +} +_mp_load_env_file "${ROOT}/config/local.env" + +# Re-apply path defaults if local.env used relative paths +export LOGO_PATH="${LOGO_PATH:-${ROOT}/assets/pirate_drift.png}" +export PIPE_PATH="${PIPE_PATH:-${ROOT}/${PIPE_NAME}}" +export STORAGE_DIR="${STORAGE_DIR:-${ROOT}/storage}" +export RUN_DIR="${RUN_DIR:-${ROOT}/run}" +export SOURCES_LIST="${SOURCES_LIST:-${ROOT}/config/sources.list}" +export SINKS_LIST="${SINKS_LIST:-${ROOT}/config/sinks.list}" diff --git a/lib/filters.sh b/lib/filters.sh new file mode 100644 index 0000000..8bfee95 --- /dev/null +++ b/lib/filters.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# filter_complex builders: logo overlay, optional delogo, blurred variant. +set -euo pipefail + +# shellcheck disable=SC1091 +source "$(dirname "${BASH_SOURCE[0]}")/common.sh" + +build_filter_complex() { + local v_in="${1:-0:v}" + local logo_in="${2:-1:v}" + local profile="${FILTER_PROFILE}" + local chain scaled delogo filter + + case "${profile}" in + nologo) + FILTER_COMPLEX="[${v_in}]fps=${FPS},scale=${WIDTH}:${HEIGHT}[ovl]" + return + ;; + blurred) + chain="[${v_in}]scale=1446:1040,crop=1280:920,scale=1446:1040,crop=1280:920,noise=alls=25:allf=t+u,avgblur=16:16" + ;; + default|*) + chain="[${v_in}]fps=${FPS},scale=${WIDTH}:${HEIGHT}" + ;; + esac + + if [[ -n "${DELOGO:-}" ]]; then + IFS=: read -r dx dy dw dh <<<"${DELOGO}" + chain="${chain},delogo=x=${dx}:y=${dy}:w=${dw}:h=${dh}:show=0" + fi + + scaled="${chain}[scaled]" + filter="[${logo_in}][scaled]scale2ref=(${WIDTH}/${HEIGHT})*ih/8:ih/8[wm][base];" + filter+="[wm]setsar=1[wmsar];[base][wmsar]overlay=x=main_w-overlay_w-(main_w*0.01):y=main_h*0.01[ovl]" + FILTER_COMPLEX="${scaled};${filter}" +} + +map_video_out() { + echo '[ovl]' +} + +map_audio_out() { + local audio_input="${AUDIO_MAP_INPUT:-0}" + echo "${audio_input}:a:0" +} diff --git a/lib/logging.sh b/lib/logging.sh new file mode 100644 index 0000000..b64ce5f --- /dev/null +++ b/lib/logging.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +log() { printf '[%s] %s\n' "$(date '+%H:%M:%S')" "$*" >&2; } +warn() { log "WARN: $*"; } +die() { log "ERROR: $*"; exit 1; } diff --git a/lib/outputs.sh b/lib/outputs.sh new file mode 100644 index 0000000..7dc6099 --- /dev/null +++ b/lib/outputs.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Tee muxer targets: local file, named pipe, HLS dir, RTMP sinks list. +set -euo pipefail + +# shellcheck disable=SC1091 +source "$(dirname "${BASH_SOURCE[0]}")/common.sh" + +read_sinks() { + SINKS=() + [[ -f "${SINKS_LIST}" ]] || return 0 + while IFS= read -r line; do + line="${line%%#*}" + line="$(echo "${line}" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + [[ -n "${line}" ]] && SINKS+=("${line}") + done < "${SINKS_LIST}" +} + +pipe_tee_entry() { + local pipe="$1" + printf "[select='v:0,a':f=flv:onfail=ignore:use_fifo=1:fifo_options='fifo_format=flv:drop_pkts_on_overflow=1:attempt_recovery=1:recovery_wait_time=1']%s" "${pipe}" +} + +file_tee_entry() { + local path="$1" + local fmt="${RECORD_FORMAT}" + case "${fmt}" in + mkv) printf "[select='v:0,a':f=matroska:onfail=ignore]%s" "${path}" ;; + flv|*) printf "[select='v:0,a':f=flv:onfail=ignore]%s" "${path}" ;; + esac +} + +hls_tee_entry() { + local dir="$1" + mkdir -p "${dir}" + printf "[select='v:0,a':f=hls:onfail=ignore:hls_time=2:hls_list_size=6:hls_flags=delete_segments+append_list]%s/index.m3u8" "${dir}" +} + +rtmp_tee_entry() { + local url="$1" + printf "[select='v:0,a':f=flv:onfail=ignore]%s" "${url}" +} + +# Encode stage: record + pipe (+ optional HLS). RTMP fanout is copy stage. +build_encode_tee_spec() { + local record="$1" + local pipe="$2" + local parts=() + parts+=("$(file_tee_entry "${record}")") + parts+=("$(pipe_tee_entry "${pipe}")") + if [[ -n "${HLS_OUTPUT_DIR}" ]]; then + parts+=("$(hls_tee_entry "${HLS_OUTPUT_DIR}")") + fi + local IFS='|' + echo "${parts[*]}" +} + +# Fanout stage: copy from pipe to all sinks (no re-encode). +build_fanout_tee_spec() { + read_sinks + local parts=() + local s + for s in "${SINKS[@]}"; do + parts+=("$(rtmp_tee_entry "${s}")") + done + [[ ${#parts[@]} -gt 0 ]] || die "no sinks in ${SINKS_LIST}" + local IFS='|' + echo "${parts[*]}" +} diff --git a/lib/source.sh b/lib/source.sh new file mode 100644 index 0000000..2d1868a --- /dev/null +++ b/lib/source.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# Resolve pipeline input from CLI arg, sources.list index, or env SOURCE_URL. +set -euo pipefail + +# shellcheck disable=SC1091 +source "$(dirname "${BASH_SOURCE[0]}")/common.sh" + +SOURCE_URL="" +SOURCE_FMT="" +SOURCE_VIDEO_STREAM="v:0" +SOURCE_AUDIO_STREAM="a:0" +AUDIO_INPUT_ARGS=() +USE_YTDLP=0 + +is_page_url() { + local url="$1" + [[ "$url" =~ ^https?:// ]] || return 1 + [[ "$url" =~ \.(m3u8|mpd|mp4|mkv|mov|ts|flv)(\?|$) ]] && return 1 + [[ -f "$url" ]] && return 1 + return 0 +} + +resolve_source() { + local arg="${1:-}" + SOURCE_URL="${SOURCE_URL:-}" + SOURCE_FMT="" + + if [[ -n "${SOURCE_URL}" ]]; then + : + elif [[ -n "${arg}" && -f "${arg}" && "${arg}" != *".list" ]]; then + SOURCE_URL="${arg}" + elif [[ -n "${arg}" && "${arg}" =~ ^https?:// ]]; then + SOURCE_URL="${arg}" + elif [[ -n "${arg}" && "${arg}" =~ ^[0-9]+$ ]]; then + if [[ ! -f "${SOURCES_LIST}" ]]; then + die "sources list missing: ${SOURCES_LIST} (copy sources.example.list)" + fi + local line + line="$(sed -n "${arg}p" "${SOURCES_LIST}" | sed 's/#.*//' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + [[ -n "${line}" ]] || die "no source on line ${arg} of ${SOURCES_LIST}" + SOURCE_URL="$(awk '{print $1}' <<<"${line}")" + SOURCE_FMT="$(awk '{print $2}' <<<"${line}")" + elif [[ -f "${SOURCES_LIST}" ]]; then + local line + line="$(grep -v '^[[:space:]]*#' "${SOURCES_LIST}" | grep -v '^[[:space:]]*$' | head -1)" + SOURCE_URL="$(awk '{print $1}' <<<"${line}")" + SOURCE_FMT="$(awk '{print $2}' <<<"${line}")" + else + die "usage: pass URL, file path, or 1-based index into ${SOURCES_LIST}" + fi + + [[ -n "${SOURCE_URL}" ]] || die "empty source URL" + + if [[ -n "${AUDIO_SOURCE:-}" ]]; then + read_rate_flag + AUDIO_INPUT_ARGS=("${RE_FLAG[@]}" -thread_queue_size "${THREAD_QUEUE_SIZE}" -i "${AUDIO_SOURCE}") + SOURCE_AUDIO_STREAM="a:0" + fi + + if [[ "${YTDLP_ENABLED}" == "1" ]] && is_page_url "${SOURCE_URL}"; then + USE_YTDLP=1 + fi + + echo "${SOURCE_URL}" > "${ROOT}/.current_source" + log "source: ${SOURCE_URL} (ytdlp=${USE_YTDLP})" +} + +ytdlp_auth_args() { + local auth="${ROOT}/config/yt-dlp.auth.env" + if [[ -f "${auth}" ]]; then + # shellcheck disable=SC1090 + source "${auth}" + fi + local args=() + [[ -n "${YTDLP_USERNAME:-}" ]] && args+=(--username "${YTDLP_USERNAME}") + [[ -n "${YTDLP_PASSWORD:-}" ]] && args+=(--password "${YTDLP_PASSWORD}") + [[ -n "${YTDLP_NETRC:-}" ]] && args+=(--netrc) + [[ -n "${YTDLP_EXTRA_ARGS:-}" ]] && args+=(${YTDLP_EXTRA_ARGS}) + printf '%s\n' "${args[@]}" +} + +build_ytdlp_pipeline() { + local fmt_args=() + [[ -n "${SOURCE_FMT}" ]] && fmt_args=(-f "${SOURCE_FMT}") + mapfile -t auth < <(ytdlp_auth_args) + + printf '%s\0' "${YTDLP_BIN}" \ + -o - --no-part --no-live-from-start --no-write-subs --no-embed-subs \ + --remux-video mkv --merge-output-format mkv --no-audio-multistreams \ + --geo-bypass --ignore-errors --hls-use-mpegts \ + "${auth[@]}" \ + "${fmt_args[@]}" \ + "${SOURCE_URL}" +} + +# Returns ffmpeg input argv: either direct -i URL or pipe from yt-dlp. +build_primary_input() { + read_rate_flag + if [[ "${USE_YTDLP}" == "1" ]]; then + require_cmd "${YTDLP_BIN}" + printf '%s\0' pipe + printf '%s\0' -thread_queue_size "${THREAD_QUEUE_SIZE}" -i pipe:0 + return + fi + + local url="${SOURCE_URL}" + if [[ "${url}" == *.jpg || "${url}" == *.jpeg || "${url}" == *.png ]]; then + printf '%s\0' direct + printf '%s\0' "${RE_FLAG[@]}" -thread_queue_size "${THREAD_QUEUE_SIZE}" \ + -loop 1 -i "${url}" -f lavfi -i anullsrc=channel_layout=stereo:sample_rate=48000 + return + fi + + printf '%s\0' direct + printf '%s\0' "${RE_FLAG[@]}" -thread_queue_size "${THREAD_QUEUE_SIZE}" \ + -reconnect_on_network_error 1 -reconnect_streamed 1 -reconnect_delay_max 5 \ + -i "${url}" +} diff --git a/nginx/hls-location.conf b/nginx/hls-location.conf new file mode 100644 index 0000000..157227e --- /dev/null +++ b/nginx/hls-location.conf @@ -0,0 +1,14 @@ +# Snippet for Alpine BE nginx (apps-port80.conf). +# Serves HLS segments written by streaming_v2 when HLS_OUTPUT_DIR is set. +# Replace @HLS_ROOT@ with the absolute storage path (e.g. /var/www/ac/media-pipelines/hls). + +location ^~ /app/androidcast_project/media/hls/ { + alias @HLS_ROOT@/; + add_header Cache-Control "no-cache"; + add_header Access-Control-Allow-Origin "*"; + types { + application/vnd.apple.mpegurl m3u8; + video/mp2t ts; + } + default_type application/vnd.apple.mpegurl; +} diff --git a/storage/.gitkeep b/storage/.gitkeep new file mode 100644 index 0000000..e69de29