/dev/null')); return $which !== '' ? $which : 'docker'; } public static function dockerAvailable(): bool { $bin = escapeshellarg(self::dockerBinary()); return trim((string) shell_exec("{$bin} info >/dev/null 2>&1 && echo ok")) === 'ok'; } public static function dockerCheckDetail(): string { $bin = self::dockerBinary(); $sock = '/var/run/docker.sock'; $parts = ['bin=' . $bin]; if (!is_executable($bin)) { $parts[] = 'bin_not_executable'; } if (!is_readable($sock)) { $parts[] = 'sock_not_readable'; } $parts[] = self::dockerAvailable() ? 'info_ok' : 'info_failed'; return implode(', ', $parts); } public static function generateBuildCode(): string { return 'bld-' . gmdate('Ymd-His') . '-' . substr(bin2hex(random_bytes(3)), 0, 6); } public static function resolvePipelineYaml(array $params): string { $candidates = array_filter([ (string) cfg('build.pipeline_config', ''), rtrim((string) cfg('build.repo_root', ''), '/') . '/build.config.yml', ]); foreach ($candidates as $path) { if ($path !== '' && is_file($path)) { $text = (string) file_get_contents($path); if ($text !== '') { return $text; } } } return self::generatePipelineYaml($params); } public static function generatePipelineYaml(array $params): string { $gitRef = (string) ($params['git_ref'] ?? 'next'); $gradle = (string) ($params['gradle_task'] ?? 'assembleDebug'); $channel = (string) ($params['ota_channel'] ?? 'staging'); $runTests = !empty($params['run_tests']) ? 'true' : 'false'; $runNative = !empty($params['run_native']) ? 'true' : 'false'; $runApk = !empty($params['run_apk']) ? 'true' : 'false'; $autoOta = !empty($params['auto_ota']) ? 'true' : 'false'; $ciVer = (string) cfg('build.ci_version', '00.01.00.1000'); return << $buildCode, 'status' => 'queued', 'phase' => 'queued', 'git_ref' => $params['git_ref'] ?? null, 'git_sha' => $params['git_sha'] ?? null, 'branch' => $params['branch'] ?? ($params['git_ref'] ?? null), 'pipeline_yaml' => $params['pipeline_yaml'] ?? null, 'params_json' => json_encode($params, JSON_UNESCAPED_SLASHES), 'dockerfile_version' => cfg('build.ci_version', '00.01.00.1000'), 'builder_id' => gethostname() ?: 'builder', 'ota_channel' => $params['ota_channel'] ?? 'staging', 'auto_ota' => !empty($params['auto_ota']), 'auto_deploy' => !empty($params['auto_deploy']), 'created_by_user_id' => $userId, ]); $dir = self::artifactDir($id); if (!is_dir($dir) && !mkdir($dir, 0775, true) && !is_dir($dir)) { throw new RuntimeException('Cannot create artifact dir: ' . $dir); } $logPath = $dir . '/build.log'; BuildRepository::update($id, ['log_path' => $logPath]); if (!self::dockerAvailable()) { $detail = self::dockerCheckDetail(); self::failBuild( $id, 'Docker daemon not available (' . $detail . '). ' . 'Ensure docker service is running and PHP-FPM user (nginx) is in group docker, then restart php-fpm.', $logPath ); return BuildRepository::getById($id) ?? ['id' => $id, 'build_code' => $buildCode, 'status' => 'failed']; } $phase = !empty($params['ssh_debug']) ? 'ssh_debug' : 'docker'; $runner = (string) cfg('build.runner_script', '/workspace/scripts/docker-build-runner.sh'); $repo = (string) cfg('build.repo_root', '/workspace'); $gitRemote = trim((string) shell_exec( 'git -c safe.directory=' . escapeshellarg($repo) . ' -C ' . escapeshellarg($repo) . ' remote get-url origin 2>/dev/null' )); $mobileSub = trim((string) cfg('build.mobile_subdir', 'ac-mobile-android')); $scriptsDir = trim((string) cfg('build.scripts_dir', dirname((string) cfg('build.runner_script', '')))); $env = [ 'BUILD_ID=' . $id, 'BUILD_LOG_FILE=' . escapeshellarg($logPath), 'BUILD_LOG_STDOUT_ONLY=1', 'BUILD_ISOLATE_SOURCE=' . (cfg('build.isolate_source', true) ? '1' : '0'), 'BUILD_GIT_DEPTH=' . max(1, (int) cfg('build.git_clone_depth', 1)), 'BUILD_WORK_DIR=' . escapeshellarg($repo), 'BUILD_OUT_DIR=' . escapeshellarg($dir), 'BUILD_MOBILE_SUBDIR=' . escapeshellarg($mobileSub), 'BUILD_SCRIPTS_DIR=' . escapeshellarg($scriptsDir !== '' ? $scriptsDir : dirname($runner)), 'BUILD_DOCKERFILE=' . escapeshellarg((string) cfg('build.dockerfile', '')), 'ANDROIDCAST_CI_VERSION=' . escapeshellarg((string) cfg('build.ci_version', '00.01.00.1000')), 'GIT_REF=' . escapeshellarg((string) ($params['git_ref'] ?? '')), 'GIT_SHA=' . escapeshellarg((string) ($params['git_sha'] ?? '')), 'GIT_REMOTE=' . escapeshellarg($gitRemote), 'GRADLE_TASK=' . escapeshellarg((string) ($params['gradle_task'] ?? 'assembleDebug')), 'RUN_UNIT_TESTS=' . (!empty($params['run_tests']) ? '1' : '0'), 'RUN_NATIVE=' . (!empty($params['run_native']) ? '1' : '0'), 'RUN_APK=' . (!empty($params['run_apk']) ? '1' : '0'), 'OTA_BASE_URL=' . escapeshellarg((string) ($params['ota_base_url'] ?? cfg('build.ota_base_url', ''))), 'OTA_CHANNEL=' . escapeshellarg((string) ($params['ota_channel'] ?? 'staging')), 'AUTO_OTA=' . (!empty($params['auto_ota']) ? '1' : '0'), ]; if (!empty($params['resume_from'])) { $env[] = 'BUILD_RESUME_FROM=' . escapeshellarg((string) $params['resume_from']); } if (!empty($params['reuse_build_id'])) { $reuseSrc = self::artifactDir((int) $params['reuse_build_id']) . '/src'; $env[] = 'BUILD_REUSE_SRC_DIR=' . escapeshellarg($reuseSrc); } if (!empty($params['ssh_debug'])) { $env[] = 'BUILD_SSH_DEBUG=1'; } $cmd = implode(' ', $env) . ' nohup ' . escapeshellarg($runner) . ' >> ' . escapeshellarg($logPath) . ' 2>&1 & echo $!'; $pid = trim((string) shell_exec($cmd)); if ($pid === '' || !ctype_digit($pid)) { self::failBuild($id, 'Build runner failed to start', $logPath); return BuildRepository::getById($id) ?? ['id' => $id, 'build_code' => $buildCode, 'status' => 'failed']; } file_put_contents($dir . '/runner.pid', $pid); BuildRepository::update($id, [ 'status' => 'running', 'phase' => $phase, 'started_at' => gmdate('Y-m-d H:i:s'), ]); return BuildRepository::getById($id) ?? ['id' => $id, 'build_code' => $buildCode]; } public static function failBuild(int $id, string $message, ?string $logPath = null): void { $safe = BuildErrorSanitizer::sanitize($message); if ($logPath) { file_put_contents($logPath, "\n[builder] ERROR: {$safe}\n", FILE_APPEND); } BuildRepository::update($id, [ 'status' => 'failed', 'phase' => 'error', 'error_message' => substr($safe, 0, 512), 'finished_at' => gmdate('Y-m-d H:i:s'), ]); $build = BuildRepository::getById($id); if ($build) { BuildNotifier::onFailure($id, $build, $safe); } } public static function stopBuild(int $id): bool { $build = BuildRepository::getById($id); if (!$build) { return false; } $status = (string) ($build['status'] ?? ''); if ($status === 'queued') { $dir = self::artifactDir($id); $logPath = $build['log_path'] ?? ($dir . '/build.log'); if ($logPath) { file_put_contents($logPath, "\n[builder] Stopped while queued\n", FILE_APPEND); } BuildRepository::update($id, [ 'status' => 'cancelled', 'phase' => 'stopped', 'error_message' => 'Stopped while queued', 'finished_at' => gmdate('Y-m-d H:i:s'), ]); return true; } if ($status !== 'running') { return false; } $dir = self::artifactDir($id); $pidFile = $dir . '/runner.pid'; $pid = is_file($pidFile) ? trim((string) file_get_contents($pidFile)) : ''; if ($pid !== '' && ctype_digit($pid)) { shell_exec('kill ' . escapeshellarg($pid) . ' 2>/dev/null'); } self::stopDebugContainer($id); $logPath = $build['log_path'] ?? ($dir . '/build.log'); if ($logPath) { file_put_contents($logPath, "\n[builder] Stopped by operator\n", FILE_APPEND); } BuildRepository::update($id, [ 'status' => 'cancelled', 'phase' => 'stopped', 'error_message' => 'Stopped by operator', 'finished_at' => gmdate('Y-m-d H:i:s'), ]); return true; } /** @return array */ public static function rerunBuild(int $id, ?array $overrideParams = null, ?int $userId = null): array { $build = BuildRepository::getById($id); if (!$build) { throw new InvalidArgumentException('build_not_found'); } $params = json_decode((string) ($build['params_json'] ?? '{}'), true); if (!is_array($params)) { $params = []; } if ($overrideParams) { $params = array_merge($params, $overrideParams); } if (empty($params['git_ref'])) { $params['git_ref'] = $build['git_ref'] ?? $build['branch'] ?? 'next'; } if (!isset($params['rerun_of'])) { $params['rerun_of'] = $id; } return self::enqueue($params, $userId); } /** Stop an in-flight build (if any) and enqueue a fresh run with the same params. */ public static function forceRerun(int $id, ?int $userId = null): array { $build = BuildRepository::getById($id); if (!$build) { throw new InvalidArgumentException('build_not_found'); } $status = (string) ($build['status'] ?? ''); if (in_array($status, ['running', 'queued'], true)) { self::stopBuild($id); } return self::rerunBuild($id, null, $userId); } /** CircleCI-style: resume from the step that failed (git → docker-build → docker-run). */ public static function detectFailedStep(int $buildId): string { $build = BuildRepository::getById($buildId); if ($build) { $err = (string) ($build['error_message'] ?? ''); $started = (string) ($build['started_at'] ?? ''); if ($started === '' && str_contains($err, 'Docker daemon not available')) { return 'docker-setup'; } } $dir = self::artifactDir($buildId); $logTail = ''; $logPath = $dir . '/build.log'; if (is_file($logPath)) { $size = filesize($logPath) ?: 0; $logTail = (string) file_get_contents($logPath, false, null, max(0, $size - 65536)); } $hasRun = is_file($dir . '/docker-run.log') && (filesize($dir . '/docker-run.log') ?: 0) > 0; $hasBuild = is_file($dir . '/docker-build.log') && (filesize($dir . '/docker-build.log') ?: 0) > 0; if ($hasRun && !is_file($dir . '/android_cast-latest.apk')) { return 'docker-run'; } if ($hasBuild || str_contains($logTail, '[runner] step docker-build')) { if (!str_contains($logTail, '[runner] step docker-run') && !str_contains($logTail, '[runner] finished')) { return 'docker-build'; } $buildStep = $hasBuild ? (string) file_get_contents($dir . '/docker-build.log') : ''; if (str_contains($logTail, 'ERROR: failed to build') || str_contains($buildStep, 'ERROR: failed to build')) { return 'docker-build'; } } if (str_contains($logTail, 'isolate:') || str_contains($logTail, 'syncing shared checkout') || str_contains($logTail, 'dubious ownership') || is_file($dir . '/git-clone.log')) { return 'git'; } return $hasRun ? 'docker-run' : ($hasBuild ? 'docker-build' : 'git'); } /** @return array */ public static function rerunFromFailed(int $id, ?int $userId = null, bool $sshDebug = false): array { $build = BuildRepository::getById($id); if (!$build) { throw new InvalidArgumentException('build_not_found'); } $status = (string) ($build['status'] ?? ''); if (!in_array($status, ['failed', 'cancelled'], true)) { throw new InvalidArgumentException('rerun_from_failed_requires_failed_build'); } $step = self::detectFailedStep($id); return self::rerunBuild($id, [ 'resume_from' => $step, 'reuse_build_id' => $id, 'ssh_debug' => $sshDebug, 'rerun_of' => $id, ], $userId); } public static function debugContainerName(int $buildId): string { return 'androidcast-bld-' . $buildId; } public static function debugContainerAlive(int $buildId): bool { $bin = escapeshellarg(self::dockerBinary()); $name = escapeshellarg(self::debugContainerName($buildId)); return trim((string) shell_exec("{$bin} inspect -f '{{.State.Running}}' {$name} 2>/dev/null")) === 'true'; } public static function stopDebugContainer(int $buildId): void { $bin = escapeshellarg(self::dockerBinary()); $name = escapeshellarg(self::debugContainerName($buildId)); shell_exec("{$bin} rm -f {$name} 2>/dev/null"); } public static function runnerAlive(int $buildId): bool { $pidFile = self::artifactDir($buildId) . '/runner.pid'; $pid = is_file($pidFile) ? trim((string) file_get_contents($pidFile)) : ''; if ($pid === '' || !ctype_digit($pid)) { return false; } $out = trim((string) shell_exec('kill -0 ' . escapeshellarg($pid) . ' 2>/dev/null && echo alive')); return $out === 'alive'; } public static function reconcileRunningBuilds(): void { BuildRepository::ensureSchema(); $stmt = Database::pdo()->query("SELECT id, log_path FROM builds WHERE status = 'running'"); if (!$stmt) { return; } foreach ($stmt->fetchAll(PDO::FETCH_ASSOC) as $row) { $id = (int) ($row['id'] ?? 0); if ($id <= 0) { continue; } $dir = self::artifactDir($id); $logPath = (string) ($row['log_path'] ?? ($dir . '/build.log')); if (is_file($dir . '/android_cast-latest.apk')) { self::finalizeFromArtifacts($id); continue; } $tail = ''; if (is_file($logPath)) { $size = filesize($logPath) ?: 0; $tail = (string) file_get_contents($logPath, false, null, max(0, $size - 8192)); } $modeFile = $dir . '/runner.mode'; if (is_file($modeFile) && trim((string) file_get_contents($modeFile)) === 'ssh_debug') { if (self::debugContainerAlive($id)) { BuildRepository::update($id, ['status' => 'running', 'phase' => 'ssh_debug']); continue; } self::failBuild($id, 'SSH debug container ended', $logPath); continue; } if (self::runnerAlive($id)) { continue; } if (str_contains($tail, 'Cannot connect to the Docker daemon') || !self::dockerAvailable()) { self::failBuild($id, 'Docker daemon not available', $logPath); } elseif (str_contains($tail, '[runner] finished')) { self::finalizeFromArtifacts($id); } elseif ($tail !== '') { $hint = self::summarizeLogFailure($logPath); $msg = $hint !== '' ? $hint : 'Build runner exited unexpectedly'; self::failBuild($id, $msg, $logPath); } else { self::failBuild($id, 'Build runner never started', $logPath); } } } public static function reconcileQueuedBuilds(): void { BuildRepository::ensureSchema(); $stmt = Database::pdo()->query("SELECT id, log_path, created_at FROM builds WHERE status = 'queued'"); if (!$stmt) { return; } foreach ($stmt->fetchAll(PDO::FETCH_ASSOC) as $row) { $id = (int) ($row['id'] ?? 0); if ($id <= 0) { continue; } $logPath = (string) ($row['log_path'] ?? (self::artifactDir($id) . '/build.log')); if (self::runnerAlive($id)) { BuildRepository::update($id, [ 'status' => 'running', 'phase' => 'docker', 'started_at' => gmdate('Y-m-d H:i:s'), ]); continue; } $created = strtotime((string) ($row['created_at'] ?? '')) ?: 0; if ($created > 0 && $created < time() - 120) { self::failBuild($id, 'Build stuck in queued (runner never started)', $logPath); } } } /** Last lines / markers from build.log for reconcile error_message (max 512 chars). */ public static function summarizeLogFailure(?string $logPath): string { if (!$logPath || !is_file($logPath)) { return ''; } $size = filesize($logPath) ?: 0; $chunk = (string) file_get_contents($logPath, false, null, max(0, $size - 32768)); if ($chunk === '') { return ''; } $lines = preg_split('/\r\n|\r|\n/', $chunk) ?: []; $markers = []; foreach ($lines as $line) { $trim = trim($line); if ($trim === '') { continue; } if (preg_match('/\[runner\] ERROR:|\[builder\] ERROR:|fatal:|BUILD FAILED|Cannot connect to the Docker daemon/i', $trim)) { $markers[] = $trim; } } $tailLines = array_values(array_filter($lines, static fn(string $l): bool => trim($l) !== '')); $last = array_slice($tailLines, -4); $parts = $markers !== [] ? array_slice($markers, -2) : $last; $summary = implode(' | ', $parts); if ($summary === '') { return ''; } if (strlen($summary) > 480) { $summary = '…' . substr($summary, -477); } return BuildErrorSanitizer::sanitize('Build runner exited unexpectedly: ' . $summary); } public static function readLogTail(?string $logPath, int $offset = 0, int $maxBytes = 65536): array { if (!$logPath || !is_file($logPath)) { return ['offset' => 0, 'eof' => true, 'text' => '', 'size' => 0]; } $size = filesize($logPath) ?: 0; $offset = max(0, min($offset, $size)); $fh = fopen($logPath, 'rb'); if (!$fh) { return ['offset' => $offset, 'eof' => true, 'text' => '', 'size' => $size]; } fseek($fh, $offset); $chunk = fread($fh, $maxBytes) ?: ''; $newOffset = $offset + strlen($chunk); fclose($fh); return ['offset' => $newOffset, 'eof' => $newOffset >= $size, 'text' => $chunk, 'size' => $size]; } public static function readLogFull(?string $logPath, int $maxBytes = 2097152): array { if (!$logPath || !is_file($logPath)) { return ['offset' => 0, 'eof' => true, 'text' => '', 'size' => 0, 'truncated' => false]; } $size = filesize($logPath) ?: 0; $truncated = $size > $maxBytes; $start = $truncated ? $size - $maxBytes : 0; $text = (string) file_get_contents($logPath, false, null, $start); if ($truncated) { $text = "[log truncated: showing last {$maxBytes} of {$size} bytes]\n" . $text; } return ['offset' => $size, 'eof' => true, 'text' => $text, 'size' => $size, 'truncated' => $truncated]; } public static function finalizeFromArtifacts(int $buildId): void { $dir = self::artifactDir($buildId); $infoPath = $dir . '/BUILD_INFO.json'; $status = is_file($dir . '/android_cast-latest.apk') ? 'passed' : 'failed'; $artifacts = []; if (is_file($dir . '/android_cast-latest.apk')) { $artifacts['apk'] = 'android_cast-latest.apk'; } if (is_dir($dir . '/ota/v0')) { $artifacts['ota'] = 'ota/v0'; } $versionApp = null; if (is_file($infoPath)) { $info = json_decode((string) file_get_contents($infoPath), true); if (is_array($info)) { $versionApp = $info['gitSha'] ?? null; } } $update = [ 'status' => $status, 'phase' => 'done', 'artifacts_json' => json_encode($artifacts, JSON_UNESCAPED_SLASHES), 'version_app' => $versionApp, 'finished_at' => gmdate('Y-m-d H:i:s'), ]; if ($status === 'failed') { $update['error_message'] = substr( BuildErrorSanitizer::sanitize('Build finished without APK artifact'), 0, 512 ); } BuildRepository::update($buildId, $update); if ($status === 'failed') { $build = BuildRepository::getById($buildId); if ($build) { BuildNotifier::onFailure($buildId, $build, (string) ($update['error_message'] ?? 'Build failed')); } } if (!empty($artifacts['ota']) && cfg('build.ota_mount')) { $build = BuildRepository::getById($buildId); if (!empty($build['auto_deploy'])) { self::publishOta($buildId, $dir); } } } private static function publishOta(int $buildId, string $dir): void { $mount = rtrim((string) cfg('build.ota_mount', ''), '/'); if ($mount === '' || !is_dir($mount)) { return; } $src = $dir . '/ota/v0'; if (!is_dir($src)) { return; } $build = BuildRepository::getById($buildId); $channel = (string) ($build['ota_channel'] ?? 'staging'); $dest = $mount . '/v0'; shell_exec('mkdir -p ' . escapeshellarg($dest) . ' && cp -a ' . escapeshellarg($src . '/.') . ' ' . escapeshellarg($dest . '/')); $channelJson = $src . '/ota/channel/' . $channel . '.json'; if (is_file($channelJson)) { shell_exec( 'mkdir -p ' . escapeshellarg($dest . '/ota/channel') . ' && cp -a ' . escapeshellarg($channelJson) . ' ' . escapeshellarg($dest . '/ota/channel/' . $channel . '.json') ); } self::publishBrowserApk($src, $channel); } /** Copy installable .apk to hub /v0/downloads/ for browser sideload. */ private static function publishBrowserApk(string $otaV0Dir, string $channel): void { $downloads = rtrim((string) cfg('build.downloads_mount', ''), '/'); if ($downloads === '') { $parent = dirname(rtrim((string) cfg('build.ota_mount', ''), '/')); $downloads = $parent !== '' && $parent !== '.' ? $parent . '/downloads' : '/var/www/localhost/htdocs/apps/app/androidcast_project/downloads'; } if (!is_dir($downloads) && !@mkdir($downloads, 0775, true) && !is_dir($downloads)) { return; } $apkPath = self::resolveBrowserApkPath($otaV0Dir, $channel); if ($apkPath === null || !is_file($apkPath)) { return; } $latest = $downloads . '/android_cast-latest.apk'; $versioned = $downloads . '/' . basename($apkPath); @copy($apkPath, $latest); @copy($apkPath, $versioned); @chown($latest, 'nginx'); @chgrp($latest, 'nginx'); @chown($versioned, 'nginx'); @chgrp($versioned, 'nginx'); } private static function resolveBrowserApkPath(string $otaV0Dir, string $channel): ?string { $channelFile = $otaV0Dir . '/ota/channel/' . $channel . '.json'; if (is_file($channelFile)) { $channelData = json_decode((string) file_get_contents($channelFile), true); $manifestUrl = is_array($channelData) ? (string) ($channelData['manifestUrl'] ?? '') : ''; if ($manifestUrl !== '') { $manifestPath = self::otaUrlToLocalPath($manifestUrl, $otaV0Dir); if ($manifestPath !== null && is_file($manifestPath)) { $manifest = json_decode((string) file_get_contents($manifestPath), true); if (is_array($manifest)) { $browser = (string) ($manifest['browserApkUrl'] ?? ''); if ($browser !== '') { $local = self::otaUrlToLocalPath($browser, $otaV0Dir); if ($local !== null && is_file($local)) { return $local; } } } } } } $glob = glob($otaV0Dir . '/ota/*/*/*/*.apk') ?: []; if ($glob !== []) { usort($glob, static fn(string $a, string $b): int => filemtime($b) <=> filemtime($a)); return $glob[0]; } return null; } private static function otaUrlToLocalPath(string $url, string $otaV0Dir): ?string { $path = (string) parse_url($url, PHP_URL_PATH); $prefix = '/v0/ota/'; if (!str_starts_with($path, $prefix)) { return null; } $rel = substr($path, strlen($prefix)); $local = $otaV0Dir . '/ota/' . $rel; return is_file($local) ? $local : null; } }