From f3f7e1982cdd21800f9f3b79c1dc8d427f03e0d7 Mon Sep 17 00:00:00 2001 From: cryptocommuniums-afk Date: Tue, 17 Mar 2026 09:51:47 +0800 Subject: [PATCH] Improve live camera relay buffering --- client/src/lib/changelog.ts | 85 +- client/src/lib/media.ts | 86 +- client/src/pages/LiveCamera.tsx | 2352 +++++++++++++++++++++---------- docs/CHANGELOG.md | 29 + media/main.go | 236 +++- media/main_test.go | 130 ++ tests/e2e/app.spec.ts | 75 +- tests/e2e/helpers/mockApp.ts | 748 +++++----- 8 files changed, 2536 insertions(+), 1205 deletions(-) diff --git a/client/src/lib/changelog.ts b/client/src/lib/changelog.ts index e556848..25fd828 100644 --- a/client/src/lib/changelog.ts +++ b/client/src/lib/changelog.ts @@ -8,11 +8,34 @@ export type ChangeLogEntry = { }; export const CHANGE_LOG_ENTRIES: ChangeLogEntry[] = [ + { + version: "2026.03.17-live-camera-relay-buffer", + releaseDate: "2026-03-17", + repoVersion: "63dbfd2+relay-buffer", + summary: + "实时分析同步观看改为服务端滚动视频缓存，观看端不再轮询单帧图片；media 服务同时新增最近 60 秒缓冲和 30 分钟缓存清理。", + features: [ + "live-camera owner 端的 60 秒合成录像分段现在会额外上传到 media relay 会话，观看端改为播放服务端生成的滚动 preview 视频，不再依赖 `live-frame.jpg` 单帧轮询", + "relay 会话只保留最近 60 秒分段，worker 会在新分段到达后按最新窗口重建 `preview.webm`，避免观看端继续看到旧一分钟缓存", + "超过 30 分钟无活动的 relay 会话、分段目录和公开缓存文件会自动清理，避免多端同步长期堆积无用缓存", + "实时分析 viewer 文案和占位提示同步调整为“缓冲最近 60 秒视频 / 加载缓存回放”，更贴近现在的服务端缓存播放行为", + "media preview 非归档阶段跳过 mp4 转码，Chrome 观看直接使用 webm，降低 worker 处理时延和 CPU 消耗", + ], + tests: [ + "cd media && go test ./...", + "pnpm vitest run client/src/lib/liveCamera.test.ts", + 'pnpm exec playwright test tests/e2e/app.spec.ts --grep "live camera page exposes camera startup controls|live camera starts analysis and produces scores|live camera switches into viewer mode when another device already owns analysis|live camera recovers mojibake viewer titles before rendering|live camera no longer opens viewer peer retries when server relay is active"', + "pnpm check", + "pnpm build", + "线上 smoke: 部署后确认 `https://te.hao.work/` 已提供新构建而不是旧资源版本，`/live-camera` viewer 端进入“服务端缓存同步”路径并返回正确的 JS/CSS MIME", + ], + }, { version: "2026.03.17-live-camera-preview-recovery", releaseDate: "2026-03-17", repoVersion: "06b9701", - summary: "修复实时分析页标题乱码、同步观看残留状态导致的黑屏，以及切回本机摄像头后预览无法恢复的问题。", + summary: + "修复实时分析页标题乱码、同步观看残留状态导致的黑屏，以及切回本机摄像头后预览无法恢复的问题。", features: [ "runtime 标题恢复逻辑新增更严格的乱码筛除与二次 UTF-8 解码兜底，`服...`、带替换字符的脏标题现在会优先恢复为正常中文，无法恢复时会安全回退到稳定默认标题", "同步观看退出时会完整重置 viewer 轮询、连接标记和帧版本，不再把旧 viewer 状态残留到 owner 或空闲态，避免页面继续停留在黑屏或“等待同步画面”", @@ -22,7 +45,7 @@ export const CHANGE_LOG_ENTRIES: ChangeLogEntry[] = [ tests: [ "pnpm check", "pnpm vitest run client/src/lib/liveCamera.test.ts", - "pnpm exec playwright test tests/e2e/app.spec.ts --grep \"live camera\"", + 'pnpm exec playwright test tests/e2e/app.spec.ts --grep "live camera"', "pnpm build", "线上 smoke: `curl -I https://te.hao.work/`，并检查页面源码中的 `/assets/index-*.js`、`/assets/index-*.css`、`/assets/pose-*.js` 已切换到新构建且返回正确 MIME", ], @@ -31,7 +54,8 @@ export const CHANGE_LOG_ENTRIES: ChangeLogEntry[] = [ version: "2026.03.16-live-camera-runtime-refresh", releaseDate: "2026-03-16", repoVersion: "8e9e491", - summary: "修复实时分析页偶发残留在同步观看状态、标题乱码，以及摄像头预览绑定波动导致的启动失败。", + summary: + "修复实时分析页偶发残留在同步观看状态、标题乱码，以及摄像头预览绑定波动导致的启动失败。", features: [ "live-camera 在打开拍摄引导、启用摄像头、开始分析前，都会先向服务端强制刷新 runtime 状态，避免旧的 viewer 锁残留导致本机明明已释放却仍无法启动", "同步观看标题新增乱码恢复逻辑，可自动把 UTF-8 被误按 Latin-1 显示的标题恢复成正常中文，避免出现 `服...` 一类异常标题", @@ -39,7 +63,7 @@ export const CHANGE_LOG_ENTRIES: ChangeLogEntry[] = [ "e2e mock 的媒体流补齐为带假视频轨道的流对象，并把 viewer 回归改为校验“服务端 relay、无 viewer-signal”行为，减少和旧 P2P 逻辑混淆", ], tests: [ - "pnpm exec playwright test tests/e2e/app.spec.ts --grep \"live camera page exposes camera startup controls|live camera switches into viewer mode when another device already owns analysis|live camera recovers mojibake viewer titles before rendering|live camera no longer opens viewer peer retries when server relay is active\"", + 'pnpm exec playwright test tests/e2e/app.spec.ts --grep "live camera page exposes camera startup controls|live camera switches into viewer mode when another device already owns analysis|live camera recovers mojibake viewer titles before rendering|live camera no longer opens viewer peer retries when server relay is active"', "pnpm build", "部署后线上 smoke: `https://te.hao.work/live-camera` 登录 H1 后可见空闲态“启动摄像头”入口，确认不再被残留 viewer 锁卡住；公开站点前端资源为 `assets/index-33wVjC4p.js` 与 `assets/index-tNGuStgv.css`", ], @@ -48,7 +72,8 @@ export const CHANGE_LOG_ENTRIES: ChangeLogEntry[] = [ version: "2026.03.16-live-viewer-server-relay", releaseDate: "2026-03-16", repoVersion: "bb46d26", - summary: "实时分析同步观看改为由 media 服务中转帧图，不再依赖浏览器之间的 P2P 视频连接。", + summary: + "实时分析同步观看改为由 media 服务中转帧图，不再依赖浏览器之间的 P2P 视频连接。", features: [ "owner 端现在会把带骨架、关键点和虚拟形象叠层的合成画布压缩成 JPEG 并持续上传到 media 服务", "viewer 端改为直接拉取 media 服务中的最新同步帧图，不再建立 WebRTC viewer peer 连接，因此跨网络和多端观看更稳定", @@ -65,7 +90,8 @@ export const CHANGE_LOG_ENTRIES: ChangeLogEntry[] = [ version: "2026.03.16-camera-startup-fallbacks", releaseDate: "2026-03-16", repoVersion: "a211562", - summary: "修复部分设备上摄像头因后置镜头约束、分辨率约束或麦克风不可用而直接启动失败的问题。", + summary: + "修复部分设备上摄像头因后置镜头约束、分辨率约束或麦克风不可用而直接启动失败的问题。", features: [ "live-camera 与 recorder 改为共用分级降级的摄像头请求流程，会在当前画质失败时自动降分辨率、降约束并回退到兼容镜头", "当设备不支持默认后置摄像头或当前镜头不可用时，页面会自动切换到实际可用的镜头方向，避免直接报错后卡死在未启动状态", @@ -81,7 +107,8 @@ export const CHANGE_LOG_ENTRIES: ChangeLogEntry[] = [ version: "2026.03.16-live-analysis-viewer-full-sync", releaseDate: "2026-03-16", repoVersion: "922a9fb", - summary: "多端同步观看改为按持有端快照完整渲染，另一设备可同步看到视频状态、模式、画质、虚拟形象和保存阶段信息。", + summary: + "多端同步观看改为按持有端快照完整渲染，另一设备可同步看到视频状态、模式、画质、虚拟形象和保存阶段信息。", features: [ "viewer 端现在同步显示持有端的会话标题、训练模式、设备端、拍摄视角、画质模式、虚拟形象状态和最近同步时间", "同步观看时的分析阶段、保存阶段、已完成状态也会跟随主端刷新，不再只显示本地默认状态", @@ -89,7 +116,7 @@ export const CHANGE_LOG_ENTRIES: ChangeLogEntry[] = [ "新增 viewer 同步信息卡，明确允许 1 秒级延迟，并持续显示最近心跳时间", ], tests: [ - "pnpm exec playwright test tests/e2e/app.spec.ts --grep \"live camera switches into viewer mode|viewer stream|recorder blocks\"", + 'pnpm exec playwright test tests/e2e/app.spec.ts --grep "live camera switches into viewer mode|viewer stream|recorder blocks"', "pnpm build", "部署后线上 smoke: `https://te.hao.work/` 已提供 `assets/index-HRdM3fxq.js` 与 `assets/index-tNGuStgv.css`；同账号 H1 双端登录后，移动端 owner 可开始实时分析，桌面端 `/live-camera` 自动进入同步观看并显示主端信息、同步视频流，owner 点击结束分析后 viewer 会同步进入保存阶段", ], @@ -98,7 +125,8 @@ export const CHANGE_LOG_ENTRIES: ChangeLogEntry[] = [ version: "2026.03.16-live-analysis-lock-hardening", releaseDate: "2026-03-16", repoVersion: "f9db6ef", - summary: "修复同账号多端实时分析在旧登录态下仍可重复占用摄像头的问题，补强同步观看重试、录制页占用锁，并修复部署后启动阶段长时间 502。", + summary: + "修复同账号多端实时分析在旧登录态下仍可重复占用摄像头的问题，补强同步观看重试、录制页占用锁，并修复部署后启动阶段长时间 502。", features: [ "旧用户名登录 token 即使缺少 `sid`，现在也会按 token 本身派生唯一会话标识，不再把不同设备错误识别成同一持有端", "同步观看模式新增 viewer 自动重试：当持有端刚启动推流、viewer 首次连接返回 `viewer stream not ready` 时，会自动重连而不是一直黑屏", @@ -110,7 +138,7 @@ export const CHANGE_LOG_ENTRIES: ChangeLogEntry[] = [ "curl -I https://te.hao.work/", "pnpm check", "pnpm exec vitest run server/_core/sdk.test.ts server/features.test.ts", - "pnpm exec playwright test tests/e2e/app.spec.ts --grep \"viewer mode|viewer stream|recorder blocks\"", + 'pnpm exec playwright test tests/e2e/app.spec.ts --grep "viewer mode|viewer stream|recorder blocks"', "pnpm build", "线上 smoke: H1 手机端开启实时分析后，PC 端 `/live-camera` 自动进入同步观看并显示同步画面，`/recorder` 禁止启动摄像头；结束分析后会话可正常释放", ], @@ -119,7 +147,8 @@ export const CHANGE_LOG_ENTRIES: ChangeLogEntry[] = [ version: "2026.03.16-live-analysis-runtime-migration", releaseDate: "2026-03-16", repoVersion: "2b72ef9", - summary: "修复实时分析因缺失 `live_analysis_runtime` 表导致的启动失败，并补齐迁移记录避免后续部署再次漏表。", + summary: + "修复实时分析因缺失 `live_analysis_runtime` 表导致的启动失败，并补齐迁移记录避免后续部署再次漏表。", features: [ "生产库补建 `live_analysis_runtime` 表，并补写 `__drizzle_migrations` 中缺失的 `0011_live_analysis_runtime` 记录", "仓库内 Drizzle migration journal 补齐 `0011_live_analysis_runtime` 条目，后续 `docker compose` 部署可正确感知该迁移", @@ -139,7 +168,8 @@ export const CHANGE_LOG_ENTRIES: ChangeLogEntry[] = [ version: "2026.03.16-live-camera-multidevice-viewer", releaseDate: "2026-03-16", repoVersion: "4e4122d", - summary: "实时分析新增同账号多端互斥和同步观看模式，分析持有端独占摄像头，其它端只能查看同步画面与核心识别结果。", + summary: + "实时分析新增同账号多端互斥和同步观看模式，分析持有端独占摄像头，其它端只能查看同步画面与核心识别结果。", features: [ "同一账号在 `/live-camera` 进入实时分析后，会写入按用户维度的 runtime 锁，其他设备不能重复启动摄像头或分析", "其他设备会自动进入“同步观看模式”，可订阅持有端的实时画面，并同步看到动作、评分、反馈、最近片段和归档段数", @@ -152,8 +182,8 @@ export const CHANGE_LOG_ENTRIES: ChangeLogEntry[] = [ "pnpm exec vitest run server/features.test.ts", "go test ./... && go build ./... (media)", "pnpm build", - "pnpm exec playwright test tests/e2e/app.spec.ts --grep \"live camera\"", - "pnpm exec playwright test tests/e2e/app.spec.ts --grep \"recorder flow archives a session and exposes it in videos\"", + 'pnpm exec playwright test tests/e2e/app.spec.ts --grep "live camera"', + 'pnpm exec playwright test tests/e2e/app.spec.ts --grep "recorder flow archives a session and exposes it in videos"', "curl -I https://te.hao.work/live-camera", ], }, @@ -161,7 +191,8 @@ export const CHANGE_LOG_ENTRIES: ChangeLogEntry[] = [ version: "2026.03.16-live-analysis-overlay-archive", releaseDate: "2026-03-16", repoVersion: "4fb2d09", - summary: "实时分析新增 60 秒自动归档录像，录制内容会保留骨架、关键点和虚拟形象叠层，并同步进入视频库。", + summary: + "实时分析新增 60 秒自动归档录像，录制内容会保留骨架、关键点和虚拟形象叠层，并同步进入视频库。", features: [ "实时分析开始后会自动录制合成画布，每 60 秒自动切段归档", "归档录像会保留原视频、骨架线、关键点和当前虚拟形象覆盖效果", @@ -181,17 +212,15 @@ export const CHANGE_LOG_ENTRIES: ChangeLogEntry[] = [ version: "2026.03.15-live-analysis-leave-hint", releaseDate: "2026-03-15", repoVersion: "5c2dcf2", - summary: "实时分析结束后增加离开提示，明确何时必须停留、何时可以安全关闭或切页。", + summary: + "实时分析结束后增加离开提示，明确何时必须停留、何时可以安全关闭或切页。", features: [ "分析进行中显示“不要关闭或切走页面”提示", "结束分析后保存阶段显示“请暂时停留当前页面”提示", "保存成功后明确提示“现在可以关闭浏览器或切换到其他页面”", "分析中和保存中挂接 beforeunload 提醒，减少误关页面导致的数据丢失", ], - tests: [ - "pnpm check", - "pnpm build", - ], + tests: ["pnpm check", "pnpm build"], }, { version: "2026.03.15-training-generator-collapse", @@ -204,10 +233,7 @@ export const CHANGE_LOG_ENTRIES: ChangeLogEntry[] = [ "移动端继续直接展示完整生成器，避免隐藏关键操作", "未生成计划时点击“前往生成训练计划”会自动展开并滚动到生成面板", ], - tests: [ - "pnpm check", - "pnpm build", - ], + tests: ["pnpm check", "pnpm build"], }, { version: "2026.03.15-progress-time-actions", @@ -220,10 +246,7 @@ export const CHANGE_LOG_ENTRIES: ChangeLogEntry[] = [ "展开态动作明细统一用中文动作标签展示", "提醒页通知时间统一切换为 Asia/Shanghai", ], - tests: [ - "pnpm check", - "pnpm build", - ], + tests: ["pnpm check", "pnpm build"], }, { version: "2026.03.15-session-changelog", @@ -275,7 +298,7 @@ export const CHANGE_LOG_ENTRIES: ChangeLogEntry[] = [ ], tests: [ "pnpm check", - "pnpm exec vitest run server/features.test.ts -t \"video\\\\.\"", + 'pnpm exec vitest run server/features.test.ts -t "video\\\\."', "Playwright 真实站点完成 /videos 新增-编辑-删除全链路", ], }, @@ -290,8 +313,6 @@ export const CHANGE_LOG_ENTRIES: ChangeLogEntry[] = [ "训练提醒通知", "通知历史管理", ], - tests: [ - "教程库、提醒、通知相关测试通过", - ], + tests: ["教程库、提醒、通知相关测试通过"], }, ]; diff --git a/client/src/lib/media.ts b/client/src/lib/media.ts index f2d3352..1107cb4 100644 --- a/client/src/lib/media.ts +++ b/client/src/lib/media.ts @@ -14,11 +14,7 @@ export type ArchiveStatus = | "completed" | "failed"; -export type PreviewStatus = - | "idle" - | "processing" - | "ready" - | "failed"; +export type PreviewStatus = "idle" | "processing" | "ready" | "failed"; export type MediaMarker = { id: string; @@ -33,6 +29,7 @@ export type MediaSession = { id: string; userId: string; title: string; + purpose?: "recording" | "relay"; status: MediaSessionStatus; archiveStatus: ArchiveStatus; previewStatus: PreviewStatus; @@ -64,11 +61,14 @@ export type MediaSession = { markers: MediaMarker[]; }; -const MEDIA_BASE = (import.meta.env.VITE_MEDIA_BASE_URL || "/media").replace(/\/$/, ""); +const MEDIA_BASE = (import.meta.env.VITE_MEDIA_BASE_URL || "/media").replace( + /\/$/, + "" +); const RETRYABLE_STATUS = new Set([502, 503, 504]); function sleep(ms: number) { - return new Promise((resolve) => setTimeout(resolve, ms)); + return new Promise(resolve => setTimeout(resolve, ms)); } async function request(path: string, init?: RequestInit): Promise { @@ -79,7 +79,11 @@ async function request(path: string, init?: RequestInit): Promise { const response = await fetch(`${MEDIA_BASE}${path}`, init); if (!response.ok) { const errorBody = await response.json().catch(() => ({})); - const error = new Error(errorBody.error || errorBody.message || `Media service error (${response.status})`); + const error = new Error( + errorBody.error || + errorBody.message || + `Media service error (${response.status})` + ); if (RETRYABLE_STATUS.has(response.status) && attempt < 2) { lastError = error; await sleep(400 * (attempt + 1)); @@ -89,7 +93,8 @@ async function request(path: string, init?: RequestInit): Promise { } return response.json() as Promise; } catch (error) { - lastError = error instanceof Error ? error : new Error("Media request failed"); + lastError = + error instanceof Error ? error : new Error("Media request failed"); if (attempt < 2) { await sleep(400 * (attempt + 1)); continue; @@ -109,6 +114,7 @@ export async function createMediaSession(payload: { qualityPreset: string; facingMode: string; deviceKind: string; + purpose?: "recording" | "relay"; }) { return request<{ session: MediaSession }>("/sessions", { method: "POST", @@ -117,28 +123,43 @@ export async function createMediaSession(payload: { }); } -export async function signalMediaSession(sessionId: string, payload: { sdp: string; type: string }) { - return request<{ sdp: string; type: string }>(`/sessions/${sessionId}/signal`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(payload), - }); +export async function signalMediaSession( + sessionId: string, + payload: { sdp: string; type: string } +) { + return request<{ sdp: string; type: string }>( + `/sessions/${sessionId}/signal`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + } + ); } -export async function signalMediaViewerSession(sessionId: string, payload: { sdp: string; type: string }) { - return request<{ viewerId: string; sdp: string; type: string }>(`/sessions/${sessionId}/viewer-signal`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(payload), - }); +export async function signalMediaViewerSession( + sessionId: string, + payload: { sdp: string; type: string } +) { + return request<{ viewerId: string; sdp: string; type: string }>( + `/sessions/${sessionId}/viewer-signal`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + } + ); } export async function uploadMediaLiveFrame(sessionId: string, blob: Blob) { - return request<{ session: MediaSession }>(`/sessions/${sessionId}/live-frame`, { - method: "POST", - headers: { "Content-Type": blob.type || "image/jpeg" }, - body: blob, - }); + return request<{ session: MediaSession }>( + `/sessions/${sessionId}/live-frame`, + { + method: "POST", + headers: { "Content-Type": blob.type || "image/jpeg" }, + body: blob, + } + ); } export async function uploadMediaSegment( @@ -159,7 +180,12 @@ export async function uploadMediaSegment( export async function createMediaMarker( sessionId: string, - payload: { type: string; label: string; timestampMs: number; confidence?: number } + payload: { + type: string; + label: string; + timestampMs: number; + confidence?: number; + } ) { return request<{ session: MediaSession }>(`/sessions/${sessionId}/markers`, { method: "POST", @@ -201,7 +227,11 @@ export function pickRecorderMimeType() { "video/webm;codecs=h264,opus", "video/webm", ]; - return candidates.find((candidate) => window.MediaRecorder?.isTypeSupported(candidate)) || "video/webm"; + return ( + candidates.find(candidate => + window.MediaRecorder?.isTypeSupported(candidate) + ) || "video/webm" + ); } export function pickBitrate(preset: string, isMobile: boolean) { diff --git a/client/src/pages/LiveCamera.tsx b/client/src/pages/LiveCamera.tsx index 26a4fe7..9e0f66f 100644 --- a/client/src/pages/LiveCamera.tsx +++ b/client/src/pages/LiveCamera.tsx @@ -3,21 +3,48 @@ import { trpc } from "@/lib/trpc"; import { createMediaSession, getMediaAssetUrl, - uploadMediaLiveFrame, + getMediaSession, + type MediaSession, + uploadMediaSegment, } from "@/lib/media"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; -import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; -import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; import { Input } from "@/components/ui/input"; import { Progress } from "@/components/ui/progress"; -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; import { Slider } from "@/components/ui/slider"; import { Switch } from "@/components/ui/switch"; import { formatDateTimeShanghai } from "@/lib/time"; import { toast } from "sonner"; -import { applyTrackZoom, type CameraQualityPreset, getLiveAnalysisBitrate, readTrackZoomState, requestCameraStream } from "@/lib/camera"; +import { + applyTrackZoom, + type CameraQualityPreset, + getLiveAnalysisBitrate, + readTrackZoomState, + requestCameraStream, +} from "@/lib/camera"; import { ACTION_WINDOW_FRAMES, AVATAR_PRESETS, @@ -153,29 +180,83 @@ type RuntimeSession = { snapshot: RuntimeSnapshot | null; }; -const ACTION_META: Record = { - forehand: { label: "正手挥拍", tone: "bg-emerald-500/10 text-emerald-700", accent: "bg-emerald-500" }, - backhand: { label: "反手挥拍", tone: "bg-sky-500/10 text-sky-700", accent: "bg-sky-500" }, - serve: { label: "发球", tone: "bg-amber-500/10 text-amber-700", accent: "bg-amber-500" }, - volley: { label: "截击", tone: "bg-indigo-500/10 text-indigo-700", accent: "bg-indigo-500" }, - overhead: { label: "高压", tone: "bg-rose-500/10 text-rose-700", accent: "bg-rose-500" }, - slice: { label: "切削", tone: "bg-orange-500/10 text-orange-700", accent: "bg-orange-500" }, - lob: { label: "挑高球", tone: "bg-fuchsia-500/10 text-fuchsia-700", accent: "bg-fuchsia-500" }, - unknown: { label: "未知动作", tone: "bg-slate-500/10 text-slate-700", accent: "bg-slate-500" }, +const ACTION_META: Record< + ActionType, + { label: string; tone: string; accent: string } +> = { + forehand: { + label: "正手挥拍", + tone: "bg-emerald-500/10 text-emerald-700", + accent: "bg-emerald-500", + }, + backhand: { + label: "反手挥拍", + tone: "bg-sky-500/10 text-sky-700", + accent: "bg-sky-500", + }, + serve: { + label: "发球", + tone: "bg-amber-500/10 text-amber-700", + accent: "bg-amber-500", + }, + volley: { + label: "截击", + tone: "bg-indigo-500/10 text-indigo-700", + accent: "bg-indigo-500", + }, + overhead: { + label: "高压", + tone: "bg-rose-500/10 text-rose-700", + accent: "bg-rose-500", + }, + slice: { + label: "切削", + tone: "bg-orange-500/10 text-orange-700", + accent: "bg-orange-500", + }, + lob: { + label: "挑高球", + tone: "bg-fuchsia-500/10 text-fuchsia-700", + accent: "bg-fuchsia-500", + }, + unknown: { + label: "未知动作", + tone: "bg-slate-500/10 text-slate-700", + accent: "bg-slate-500", + }, }; const SETUP_STEPS = [ - { title: "固定设备", desc: "手机或平板保持稳定，避免分析阶段发生晃动", icon: }, - { title: "保留全身", desc: "画面尽量覆盖从头到脚，便于识别重心和脚步", icon: }, - { title: "确认视角", desc: "后置摄像头优先，横屏更适合完整挥拍追踪", icon: }, - { title: "开始分析", desc: "动作会先经过 24 帧稳定窗口确认，再按连续区间聚合保存", icon: }, + { + title: "固定设备", + desc: "手机或平板保持稳定，避免分析阶段发生晃动", + icon: , + }, + { + title: "保留全身", + desc: "画面尽量覆盖从头到脚，便于识别重心和脚步", + icon: , + }, + { + title: "确认视角", + desc: "后置摄像头优先，横屏更适合完整挥拍追踪", + icon: , + }, + { + title: "开始分析", + desc: "动作会先经过 24 帧稳定窗口确认，再按连续区间聚合保存", + icon: , + }, ]; const SEGMENT_MAX_MS = 10_000; const MERGE_GAP_MS = 900; const MIN_SEGMENT_MS = 1_200; const ANALYSIS_RECORDING_SEGMENT_MS = 60_000; -const CAMERA_QUALITY_PRESETS: Record = { +const CAMERA_QUALITY_PRESETS: Record< + CameraQualityPreset, + { label: string; subtitle: string; description: string } +> = { economy: { label: "节省流量", subtitle: "540p-720p · 低码率", @@ -206,7 +287,8 @@ function distance(a?: Point, b?: Point) { function getAngle(a?: Point, b?: Point, c?: Point) { if (!a || !b || !c) return 0; - const radians = Math.atan2(c.y - b.y, c.x - b.x) - Math.atan2(a.y - b.y, a.x - b.x); + const radians = + Math.atan2(c.y - b.y, c.x - b.x) - Math.atan2(a.y - b.y, a.x - b.x); let angle = Math.abs((radians * 180) / Math.PI); if (angle > 180) angle = 360 - angle; return angle; @@ -233,7 +315,14 @@ function normalizeRuntimeTitle(value: string | null | undefined) { const punctuationCount = text.match(/[()\-_:./]/g)?.length ?? 0; const badCount = text.match(/[ÃÂÆÐÑØæåçéèêëïîôöûüœŠŽƒ€¦�]/g)?.length ?? 0; const controlCount = text.match(control)?.length ?? 0; - return (cjkCount * 3) + latinCount + whitespaceCount + punctuationCount - (badCount * 4) - (controlCount * 6); + return ( + cjkCount * 3 + + latinCount + + whitespaceCount + + punctuationCount - + badCount * 4 - + controlCount * 6 + ); }; const sanitize = (candidate: string) => { const normalized = candidate.replace(control, "").trim(); @@ -250,13 +339,19 @@ function normalizeRuntimeTitle(value: string | null | undefined) { const candidates = [trimmed]; try { - const bytes = Uint8Array.from(Array.from(trimmed).map((char) => char.charCodeAt(0) & 0xff)); + const bytes = Uint8Array.from( + Array.from(trimmed).map(char => char.charCodeAt(0) & 0xff) + ); const decoded = new TextDecoder("utf-8").decode(bytes).trim(); if (decoded && decoded !== trimmed) { candidates.push(decoded); if (suspicious.test(decoded)) { - const decodedBytes = Uint8Array.from(Array.from(decoded).map((char) => char.charCodeAt(0) & 0xff)); - const twiceDecoded = new TextDecoder("utf-8").decode(decodedBytes).trim(); + const decodedBytes = Uint8Array.from( + Array.from(decoded).map(char => char.charCodeAt(0) & 0xff) + ); + const twiceDecoded = new TextDecoder("utf-8") + .decode(decodedBytes) + .trim(); if (twiceDecoded && twiceDecoded !== decoded) { candidates.push(twiceDecoded); } @@ -266,26 +361,42 @@ function normalizeRuntimeTitle(value: string | null | undefined) { return sanitize(trimmed); } - return candidates - .map((candidate) => sanitize(candidate)) - .filter(Boolean) - .sort((left, right) => score(right) - score(left))[0] || ""; + return ( + candidates + .map(candidate => sanitize(candidate)) + .filter(Boolean) + .sort((left, right) => score(right) - score(left))[0] || "" + ); } function isMobileDevice() { if (typeof window === "undefined") return false; - return /Android|iPhone|iPad|iPod/i.test(navigator.userAgent) || window.matchMedia("(max-width: 768px)").matches; + return ( + /Android|iPhone|iPad|iPod/i.test(navigator.userAgent) || + window.matchMedia("(max-width: 768px)").matches + ); } function pickRecorderMimeType() { - const supported = typeof MediaRecorder !== "undefined" && typeof MediaRecorder.isTypeSupported === "function"; - if (supported && MediaRecorder.isTypeSupported("video/mp4;codecs=avc1.42E01E,mp4a.40.2")) { + const supported = + typeof MediaRecorder !== "undefined" && + typeof MediaRecorder.isTypeSupported === "function"; + if ( + supported && + MediaRecorder.isTypeSupported("video/mp4;codecs=avc1.42E01E,mp4a.40.2") + ) { return "video/mp4"; } - if (supported && MediaRecorder.isTypeSupported("video/webm;codecs=vp9,opus")) { + if ( + supported && + MediaRecorder.isTypeSupported("video/webm;codecs=vp9,opus") + ) { return "video/webm;codecs=vp9,opus"; } - if (supported && MediaRecorder.isTypeSupported("video/webm;codecs=vp8,opus")) { + if ( + supported && + MediaRecorder.isTypeSupported("video/webm;codecs=vp8,opus") + ) { return "video/webm;codecs=vp8,opus"; } return "video/webm"; @@ -308,7 +419,11 @@ function blobToBase64(blob: Blob) { }); } -function createSegment(action: ActionType, elapsedMs: number, frame: AnalyzedFrame): ActionSegment { +function createSegment( + action: ActionType, + elapsedMs: number, + frame: AnalyzedFrame +): ActionSegment { return { actionType: action, isUnknown: action === "unknown", @@ -325,7 +440,11 @@ function createSegment(action: ActionType, elapsedMs: number, frame: AnalyzedFra }; } -function analyzePoseFrame(landmarks: Point[], tracking: TrackingState, timestamp: number): AnalyzedFrame { +function analyzePoseFrame( + landmarks: Point[], + tracking: TrackingState, + timestamp: number +): AnalyzedFrame { const nose = landmarks[0]; const leftShoulder = landmarks[11]; const rightShoulder = landmarks[12]; @@ -345,22 +464,41 @@ function analyzePoseFrame(landmarks: Point[], tracking: TrackingState, timestamp y: ((leftHip?.y ?? 0.7) + (rightHip?.y ?? 0.7)) / 2, }; - const dtMs = tracking.prevTimestamp ? Math.max(16, timestamp - tracking.prevTimestamp) : 33; - const rightSpeed = distance(rightWrist, tracking.prevRightWrist) * (1000 / dtMs); + const dtMs = tracking.prevTimestamp + ? Math.max(16, timestamp - tracking.prevTimestamp) + : 33; + const rightSpeed = + distance(rightWrist, tracking.prevRightWrist) * (1000 / dtMs); const leftSpeed = distance(leftWrist, tracking.prevLeftWrist) * (1000 / dtMs); const hipSpeed = distance(hipCenter, tracking.prevHipCenter) * (1000 / dtMs); - const rightVerticalMotion = tracking.prevRightWrist ? tracking.prevRightWrist.y - (rightWrist?.y ?? tracking.prevRightWrist.y) : 0; + const rightVerticalMotion = tracking.prevRightWrist + ? tracking.prevRightWrist.y - (rightWrist?.y ?? tracking.prevRightWrist.y) + : 0; - const shoulderTilt = Math.abs((leftShoulder?.y ?? 0.3) - (rightShoulder?.y ?? 0.3)); + const shoulderTilt = Math.abs( + (leftShoulder?.y ?? 0.3) - (rightShoulder?.y ?? 0.3) + ); const hipTilt = Math.abs((leftHip?.y ?? 0.55) - (rightHip?.y ?? 0.55)); - const headOffset = Math.abs((nose?.x ?? 0.5) - (((leftShoulder?.x ?? 0.45) + (rightShoulder?.x ?? 0.55)) / 2)); - const kneeBend = ((getAngle(leftHip, leftKnee, leftAnkle) || 165) + (getAngle(rightHip, rightKnee, rightAnkle) || 165)) / 2; - const rightElbowAngle = getAngle(rightShoulder, rightElbow, rightWrist) || 145; + const headOffset = Math.abs( + (nose?.x ?? 0.5) - + ((leftShoulder?.x ?? 0.45) + (rightShoulder?.x ?? 0.55)) / 2 + ); + const kneeBend = + ((getAngle(leftHip, leftKnee, leftAnkle) || 165) + + (getAngle(rightHip, rightKnee, rightAnkle) || 165)) / + 2; + const rightElbowAngle = + getAngle(rightShoulder, rightElbow, rightWrist) || 145; const leftElbowAngle = getAngle(leftShoulder, leftElbow, leftWrist) || 145; const footSpread = Math.abs((leftAnkle?.x ?? 0.42) - (rightAnkle?.x ?? 0.58)); - const shoulderSpan = Math.abs((rightShoulder?.x ?? 0.56) - (leftShoulder?.x ?? 0.44)); - const wristSpread = Math.abs((rightWrist?.x ?? 0.62) - (leftWrist?.x ?? 0.38)); - const shoulderCenterX = ((leftShoulder?.x ?? 0.45) + (rightShoulder?.x ?? 0.55)) / 2; + const shoulderSpan = Math.abs( + (rightShoulder?.x ?? 0.56) - (leftShoulder?.x ?? 0.44) + ); + const wristSpread = Math.abs( + (rightWrist?.x ?? 0.62) - (leftWrist?.x ?? 0.38) + ); + const shoulderCenterX = + ((leftShoulder?.x ?? 0.45) + (rightShoulder?.x ?? 0.55)) / 2; const torsoOffset = Math.abs(shoulderCenterX - hipCenter.x); const rightForward = (rightWrist?.x ?? shoulderCenterX) - hipCenter.x; const leftForward = hipCenter.x - (leftWrist?.x ?? shoulderCenterX); @@ -393,98 +531,145 @@ function analyzePoseFrame(landmarks: Point[], tracking: TrackingState, timestamp } const posture = clamp(100 - shoulderTilt * 780 - headOffset * 640, 0, 100); - const balance = clamp(100 - hipTilt * 900 - Math.max(0, 0.16 - footSpread) * 260, 0, 100); - const footwork = clamp(45 + Math.min(36, hipSpeed * 120) + Math.max(0, 165 - kneeBend) * 0.35, 0, 100); - const consistency = clamp(visibility * 100 - Math.abs(rightSpeed - leftSpeed) * 10, 0, 100); + const balance = clamp( + 100 - hipTilt * 900 - Math.max(0, 0.16 - footSpread) * 260, + 0, + 100 + ); + const footwork = clamp( + 45 + Math.min(36, hipSpeed * 120) + Math.max(0, 165 - kneeBend) * 0.35, + 0, + 100 + ); + const consistency = clamp( + visibility * 100 - Math.abs(rightSpeed - leftSpeed) * 10, + 0, + 100 + ); const candidates: Array<{ action: ActionType; confidence: number }> = [ { action: "serve", confidence: clamp( (rightWrist && nose && rightWrist.y < nose.y ? 0.45 : 0.1) + - (rightElbow && rightShoulder && rightElbow.y < rightShoulder.y ? 0.18 : 0.04) + - clamp(contactHeight * 1.4, 0, 0.14) + - clamp((0.24 - footSpread) * 1.2, 0, 0.08) + - clamp((rightElbowAngle - 135) / 55, 0, 0.22) + - clamp(rightVerticalMotion * 4.5, 0, 0.15), + (rightElbow && rightShoulder && rightElbow.y < rightShoulder.y + ? 0.18 + : 0.04) + + clamp(contactHeight * 1.4, 0, 0.14) + + clamp((0.24 - footSpread) * 1.2, 0, 0.08) + + clamp((rightElbowAngle - 135) / 55, 0, 0.22) + + clamp(rightVerticalMotion * 4.5, 0, 0.15), 0, - 0.98, + 0.98 ), }, { action: "overhead", confidence: clamp( - (rightWrist && rightShoulder && rightWrist.y < rightShoulder.y - 0.1 ? 0.34 : 0.08) + - clamp(rightSpeed * 0.08, 0, 0.28) + - clamp((rightElbowAngle - 125) / 70, 0, 0.18), + (rightWrist && rightShoulder && rightWrist.y < rightShoulder.y - 0.1 + ? 0.34 + : 0.08) + + clamp(rightSpeed * 0.08, 0, 0.28) + + clamp((rightElbowAngle - 125) / 70, 0, 0.18), 0, - 0.92, + 0.92 ), }, { action: "forehand", confidence: clamp( (rightWrist && nose && rightWrist.x > nose.x ? 0.24 : 0.08) + - (rightForward > 0.11 ? 0.16 : 0.04) + - clamp((wristSpread - 0.2) * 0.8, 0, 0.16) + - clamp((0.08 - torsoOffset) * 1.8, 0, 0.08) + - clamp(rightSpeed * 0.12, 0, 0.28) + - clamp((rightElbowAngle - 85) / 70, 0, 0.2), + (rightForward > 0.11 ? 0.16 : 0.04) + + clamp((wristSpread - 0.2) * 0.8, 0, 0.16) + + clamp((0.08 - torsoOffset) * 1.8, 0, 0.08) + + clamp(rightSpeed * 0.12, 0, 0.28) + + clamp((rightElbowAngle - 85) / 70, 0, 0.2), 0, - 0.94, + 0.94 ), }, { action: "backhand", confidence: clamp( - ((leftWrist && nose && leftWrist.x < nose.x) || (rightWrist && nose && rightWrist.x < nose.x) ? 0.2 : 0.06) + - (leftForward > 0.1 ? 0.16 : 0.04) + - (rightWrist && hipCenter && rightWrist.x < hipCenter.x ? 0.12 : 0.02) + - clamp((wristSpread - 0.22) * 0.75, 0, 0.14) + - clamp(Math.max(leftSpeed, rightSpeed) * 0.1, 0, 0.22) + - clamp((leftElbowAngle - 85) / 70, 0, 0.18), + ((leftWrist && nose && leftWrist.x < nose.x) || + (rightWrist && nose && rightWrist.x < nose.x) + ? 0.2 + : 0.06) + + (leftForward > 0.1 ? 0.16 : 0.04) + + (rightWrist && hipCenter && rightWrist.x < hipCenter.x + ? 0.12 + : 0.02) + + clamp((wristSpread - 0.22) * 0.75, 0, 0.14) + + clamp(Math.max(leftSpeed, rightSpeed) * 0.1, 0, 0.22) + + clamp((leftElbowAngle - 85) / 70, 0, 0.18), 0, - 0.92, + 0.92 ), }, { action: "volley", confidence: clamp( - (rightWrist && rightShoulder && Math.abs(rightWrist.y - rightShoulder.y) < 0.12 ? 0.3 : 0.08) + - clamp((0.16 - Math.abs(contactHeight - 0.08)) * 1.2, 0, 0.1) + - clamp((0.22 - Math.abs((rightWrist?.x ?? 0.5) - hipCenter.x)) * 1.5, 0, 0.18) + - clamp((1.8 - rightSpeed) * 0.14, 0, 0.18), + (rightWrist && + rightShoulder && + Math.abs(rightWrist.y - rightShoulder.y) < 0.12 + ? 0.3 + : 0.08) + + clamp((0.16 - Math.abs(contactHeight - 0.08)) * 1.2, 0, 0.1) + + clamp( + (0.22 - Math.abs((rightWrist?.x ?? 0.5) - hipCenter.x)) * 1.5, + 0, + 0.18 + ) + + clamp((1.8 - rightSpeed) * 0.14, 0, 0.18), 0, - 0.88, + 0.88 ), }, { action: "slice", confidence: clamp( - (rightWrist && rightShoulder && rightWrist.y > rightShoulder.y ? 0.18 : 0.06) + - clamp((contactHeight + 0.06) * 0.7, 0, 0.08) + - clamp((tracking.prevRightWrist && rightWrist && rightWrist.y > tracking.prevRightWrist.y ? 0.18 : 0.04), 0, 0.18) + - clamp(rightSpeed * 0.08, 0, 0.24), + (rightWrist && rightShoulder && rightWrist.y > rightShoulder.y + ? 0.18 + : 0.06) + + clamp((contactHeight + 0.06) * 0.7, 0, 0.08) + + clamp( + tracking.prevRightWrist && + rightWrist && + rightWrist.y > tracking.prevRightWrist.y + ? 0.18 + : 0.04, + 0, + 0.18 + ) + + clamp(rightSpeed * 0.08, 0, 0.24), 0, - 0.82, + 0.82 ), }, { action: "lob", confidence: clamp( (rightWrist && nose && rightWrist.y < nose.y + 0.1 ? 0.22 : 0.08) + - clamp((0.18 - Math.abs(rightForward)) * 1.2, 0, 0.08) + - clamp(rightVerticalMotion * 4.2, 0, 0.28) + - clamp((0.18 - Math.abs((rightWrist?.x ?? 0.5) - hipCenter.x)) * 1.4, 0, 0.18), + clamp((0.18 - Math.abs(rightForward)) * 1.2, 0, 0.08) + + clamp(rightVerticalMotion * 4.2, 0, 0.28) + + clamp( + (0.18 - Math.abs((rightWrist?.x ?? 0.5) - hipCenter.x)) * 1.4, + 0, + 0.18 + ), 0, - 0.86, + 0.86 ), }, ]; candidates.sort((a, b) => b.confidence - a.confidence); - const topCandidate = candidates[0] ?? { action: "unknown" as ActionType, confidence: 0.2 }; - const action = topCandidate.confidence >= 0.52 ? topCandidate.action : "unknown"; + const topCandidate = candidates[0] ?? { + action: "unknown" as ActionType, + confidence: 0.2, + }; + const action = + topCandidate.confidence >= 0.52 ? topCandidate.action : "unknown"; const techniqueBase = action === "serve" || action === "overhead" @@ -496,12 +681,12 @@ function analyzePoseFrame(landmarks: Point[], tracking: TrackingState, timestamp const technique = clamp(techniqueBase + topCandidate.confidence * 8, 0, 100); const overall = clamp( posture * 0.22 + - balance * 0.18 + - technique * 0.28 + - footwork * 0.16 + - consistency * 0.16, + balance * 0.18 + + technique * 0.28 + + footwork * 0.16 + + consistency * 0.16, 0, - 100, + 100 ); const feedback: string[] = []; @@ -552,7 +737,15 @@ function analyzePoseFrame(landmarks: Point[], tracking: TrackingState, timestamp }; } -function ScoreBar({ label, value, accent }: { label: string; value: number; accent?: string }) { +function ScoreBar({ + label, + value, + accent, +}: { + label: string; + value: number; + accent?: string; +}) { return (
@@ -569,11 +762,23 @@ function ScoreBar({ label, value, accent }: { label: string; value: number; acce ); } -function getSessionBand(input: { overallScore: number; knownRatio: number; effectiveSegments: number }) { - if (input.overallScore >= 85 && input.knownRatio >= 0.72 && input.effectiveSegments >= 4) { +function getSessionBand(input: { + overallScore: number; + knownRatio: number; + effectiveSegments: number; +}) { + if ( + input.overallScore >= 85 && + input.knownRatio >= 0.72 && + input.effectiveSegments >= 4 + ) { return { label: "高质量", tone: "bg-emerald-500/10 text-emerald-700" }; } - if (input.overallScore >= 72 && input.knownRatio >= 0.55 && input.effectiveSegments >= 2) { + if ( + input.overallScore >= 72 && + input.knownRatio >= 0.55 && + input.effectiveSegments >= 2 + ) { return { label: "稳定", tone: "bg-sky-500/10 text-sky-700" }; } return { label: "待加强", tone: "bg-amber-500/10 text-amber-700" }; @@ -604,10 +809,7 @@ export default function LiveCamera() { const poseRef = useRef(null); const compositeCanvasRef = useRef(null); const broadcastSessionIdRef = useRef(null); - const viewerSessionIdRef = useRef(null); const viewerRetryTimerRef = useRef(0); - const frameRelayTimerRef = useRef(0); - const frameRelayInFlightRef = useRef(false); const runtimeIdRef = useRef(null); const heartbeatTimerRef = useRef(0); const recorderRef = useRef(null); @@ -619,6 +821,7 @@ export default function LiveCamera() { const recorderSequenceRef = useRef(0); const recorderRotateTimerRef = useRef(0); const recorderUploadQueueRef = useRef(Promise.resolve()); + const relayUploadQueueRef = useRef(Promise.resolve()); const archivedVideosRef = useRef([]); const analyzingRef = useRef(false); const animationRef = useRef(0); @@ -635,9 +838,13 @@ export default function LiveCamera() { const liveScoreRef = useRef(null); const feedbackRef = useRef([]); const durationMsRef = useRef(0); - const leaveStatusRef = useRef<"idle" | "analyzing" | "saving" | "safe" | "failed">("idle"); + const leaveStatusRef = useRef< + "idle" | "analyzing" | "saving" | "safe" | "failed" + >("idle"); const sessionModeRef = useRef("practice"); - const stabilityMetaRef = useRef(createEmptyStabilizedActionMeta()); + const stabilityMetaRef = useRef( + createEmptyStabilizedActionMeta() + ); const zoomTargetRef = useRef(1); const avatarRenderRef = useRef({ enabled: false, @@ -652,7 +859,9 @@ export default function LiveCamera() { const [sessionMode, setSessionMode] = useState("practice"); const [analyzing, setAnalyzing] = useState(false); const [saving, setSaving] = useState(false); - const [leaveStatus, setLeaveStatus] = useState<"idle" | "analyzing" | "saving" | "safe" | "failed">("idle"); + const [leaveStatus, setLeaveStatus] = useState< + "idle" | "analyzing" | "saving" | "safe" | "failed" + >("idle"); const [immersivePreview, setImmersivePreview] = useState(false); const [liveScore, setLiveScore] = useState(null); const [currentAction, setCurrentAction] = useState("unknown"); @@ -661,20 +870,24 @@ export default function LiveCamera() { const [segments, setSegments] = useState([]); const [durationMs, setDurationMs] = useState(0); const [segmentFilter, setSegmentFilter] = useState("all"); - const [qualityPreset, setQualityPreset] = useState("economy"); + const [qualityPreset, setQualityPreset] = + useState("economy"); const [zoomState, setZoomState] = useState(() => readTrackZoomState(null)); - const [stabilityMeta, setStabilityMeta] = useState(() => createEmptyStabilizedActionMeta()); + const [stabilityMeta, setStabilityMeta] = useState(() => + createEmptyStabilizedActionMeta() + ); const [avatarEnabled, setAvatarEnabled] = useState(false); const [avatarKey, setAvatarKey] = useState("gorilla"); const [avatarPrompt, setAvatarPrompt] = useState(""); const [archivedVideoCount, setArchivedVideoCount] = useState(0); const [viewerConnected, setViewerConnected] = useState(false); const [viewerError, setViewerError] = useState(""); - const [viewerFrameVersion, setViewerFrameVersion] = useState(0); + const [viewerMediaSession, setViewerMediaSession] = + useState(null); const resolvedAvatarKey = useMemo( () => resolveAvatarKeyFromPrompt(avatarPrompt, avatarKey), - [avatarKey, avatarPrompt], + [avatarKey, avatarPrompt] ); const uploadMutation = trpc.video.upload.useMutation(); @@ -689,7 +902,9 @@ export default function LiveCamera() { utils.rating.history.invalidate(); }, }); - const liveSessionsQuery = trpc.analysis.liveSessionList.useQuery({ limit: 8 }); + const liveSessionsQuery = trpc.analysis.liveSessionList.useQuery({ + limit: 8, + }); const runtimeQuery = trpc.analysis.runtimeGet.useQuery(undefined, { refetchInterval: 1000, refetchIntervalInBackground: true, @@ -699,7 +914,8 @@ export default function LiveCamera() { const runtimeReleaseMutation = trpc.analysis.runtimeRelease.useMutation(); const runtimeRole = (runtimeQuery.data?.role ?? "idle") as RuntimeRole; - const runtimeSession = (runtimeQuery.data?.runtimeSession ?? null) as RuntimeSession | null; + const runtimeSession = (runtimeQuery.data?.runtimeSession ?? + null) as RuntimeSession | null; const runtimeSnapshot = runtimeSession?.snapshot ?? null; const normalizedRuntimeTitle = normalizeRuntimeTitle(runtimeSession?.title); const normalizedSnapshotTitle = normalizeRuntimeTitle(runtimeSnapshot?.title); @@ -752,28 +968,54 @@ export default function LiveCamera() { }, [stabilityMeta]); const visibleSegments = useMemo( - () => segments.filter((segment) => !segment.isUnknown).sort((a, b) => b.startMs - a.startMs), - [segments], + () => + segments + .filter(segment => !segment.isUnknown) + .sort((a, b) => b.startMs - a.startMs), + [segments] + ); + const unknownSegments = useMemo( + () => segments.filter(segment => segment.isUnknown), + [segments] ); - const unknownSegments = useMemo(() => segments.filter((segment) => segment.isUnknown), [segments]); const filteredVisibleSegments = useMemo( - () => segmentFilter === "all" ? visibleSegments : visibleSegments.filter((segment) => segment.actionType === segmentFilter), - [segmentFilter, visibleSegments], + () => + segmentFilter === "all" + ? visibleSegments + : visibleSegments.filter( + segment => segment.actionType === segmentFilter + ), + [segmentFilter, visibleSegments] ); const viewerRecentSegments = useMemo( - () => (runtimeSnapshot?.recentSegments ?? []).filter((segment) => !segment.isUnknown), - [runtimeSnapshot?.recentSegments], + () => + (runtimeSnapshot?.recentSegments ?? []).filter( + segment => !segment.isUnknown + ), + [runtimeSnapshot?.recentSegments] ); - const displayVisibleSegments = runtimeRole === "viewer" ? viewerRecentSegments : visibleSegments; - const displayFilteredSegments = runtimeRole === "viewer" - ? (segmentFilter === "all" + const displayVisibleSegments = + runtimeRole === "viewer" ? viewerRecentSegments : visibleSegments; + const displayFilteredSegments = + runtimeRole === "viewer" + ? segmentFilter === "all" ? viewerRecentSegments - : viewerRecentSegments.filter((segment) => segment.actionType === segmentFilter)) - : filteredVisibleSegments; + : viewerRecentSegments.filter( + segment => segment.actionType === segmentFilter + ) + : filteredVisibleSegments; const actionStats = useMemo(() => { - const totals = new Map(); + const totals = new Map< + ActionType, + { + count: number; + durationMs: number; + averageScore: number; + averageConfidence: number; + } + >(); - displayVisibleSegments.forEach((segment) => { + displayVisibleSegments.forEach(segment => { const current = totals.get(segment.actionType) ?? { count: 0, durationMs: 0, @@ -784,12 +1026,21 @@ export default function LiveCamera() { totals.set(segment.actionType, { count: nextCount, durationMs: current.durationMs + segment.durationMs, - averageScore: ((current.averageScore * current.count) + segment.score) / nextCount, - averageConfidence: ((current.averageConfidence * current.count) + segment.confidenceAvg) / nextCount, + averageScore: + (current.averageScore * current.count + segment.score) / nextCount, + averageConfidence: + (current.averageConfidence * current.count + segment.confidenceAvg) / + nextCount, }); }); - const totalDuration = Math.max(1, displayVisibleSegments.reduce((sum, segment) => sum + segment.durationMs, 0)); + const totalDuration = Math.max( + 1, + displayVisibleSegments.reduce( + (sum, segment) => sum + segment.durationMs, + 0 + ) + ); return Array.from(totals.entries()) .map(([actionType, value]) => ({ actionType, @@ -799,56 +1050,79 @@ export default function LiveCamera() { .sort((a, b) => b.durationMs - a.durationMs); }, [displayVisibleSegments]); const bestSegment = useMemo( - () => displayVisibleSegments.reduce((best, segment) => { - if (!best) return segment; - return segment.score > best.score ? segment : best; - }, null), - [displayVisibleSegments], + () => + displayVisibleSegments.reduce((best, segment) => { + if (!best) return segment; + return segment.score > best.score ? segment : best; + }, null), + [displayVisibleSegments] ); - const totalDisplaySegments = runtimeRole === "viewer" - ? (runtimeSnapshot?.visibleSegments ?? displayVisibleSegments.length) + (runtimeSnapshot?.unknownSegments ?? 0) - : segments.length; - const knownRatio = totalDisplaySegments > 0 ? displayVisibleSegments.length / totalDisplaySegments : 0; + const totalDisplaySegments = + runtimeRole === "viewer" + ? (runtimeSnapshot?.visibleSegments ?? displayVisibleSegments.length) + + (runtimeSnapshot?.unknownSegments ?? 0) + : segments.length; + const knownRatio = + totalDisplaySegments > 0 + ? displayVisibleSegments.length / totalDisplaySegments + : 0; const sessionBand = useMemo( - () => getSessionBand({ - overallScore: (runtimeRole === "viewer" ? runtimeSnapshot?.liveScore?.overall : liveScore?.overall) || 0, + () => + getSessionBand({ + overallScore: + (runtimeRole === "viewer" + ? runtimeSnapshot?.liveScore?.overall + : liveScore?.overall) || 0, + knownRatio, + effectiveSegments: displayVisibleSegments.length, + }), + [ + displayVisibleSegments.length, knownRatio, - effectiveSegments: displayVisibleSegments.length, - }), - [displayVisibleSegments.length, knownRatio, liveScore?.overall, runtimeRole, runtimeSnapshot?.liveScore?.overall], + liveScore?.overall, + runtimeRole, + runtimeSnapshot?.liveScore?.overall, + ] ); const refreshRuntimeState = useCallback(async () => { const result = await runtimeQuery.refetch(); return { role: (result.data?.role ?? runtimeRole) as RuntimeRole, - runtimeSession: (result.data?.runtimeSession ?? runtimeSession) as RuntimeSession | null, + runtimeSession: (result.data?.runtimeSession ?? + runtimeSession) as RuntimeSession | null, }; }, [runtimeQuery, runtimeRole, runtimeSession]); useEffect(() => { - navigator.mediaDevices?.enumerateDevices().then((devices) => { - const cameras = devices.filter((device) => device.kind === "videoinput"); - setHasMultipleCameras(cameras.length > 1); - }).catch(() => undefined); + navigator.mediaDevices + ?.enumerateDevices() + .then(devices => { + const cameras = devices.filter(device => device.kind === "videoinput"); + setHasMultipleCameras(cameras.length > 1); + }) + .catch(() => undefined); }, []); - const bindLocalPreview = useCallback(async (providedStream?: MediaStream | null) => { - const stream = providedStream || streamRef.current; - const video = videoRef.current; - if (!stream || !video) { - return false; - } + const bindLocalPreview = useCallback( + async (providedStream?: MediaStream | null) => { + const stream = providedStream || streamRef.current; + const video = videoRef.current; + if (!stream || !video) { + return false; + } - if (video.srcObject !== stream) { - video.srcObject = stream; - } - video.muted = true; - video.defaultMuted = true; - video.playsInline = true; - await video.play().catch(() => undefined); - return video.srcObject === stream; - }, []); + if (video.srcObject !== stream) { + video.srcObject = stream; + } + video.muted = true; + video.defaultMuted = true; + video.playsInline = true; + await video.play().catch(() => undefined); + return video.srcObject === stream; + }, + [] + ); useEffect(() => { if (!cameraActive || !streamRef.current || runtimeRole === "viewer") return; @@ -859,16 +1133,22 @@ export default function LiveCamera() { const video = videoRef.current; const stream = streamRef.current; if (!video || !stream) return; - if (video.srcObject !== stream || video.videoWidth === 0 || video.paused) { + if ( + video.srcObject !== stream || + video.videoWidth === 0 || + video.paused + ) { void bindLocalPreview(stream); } }; ensurePreview(); - const timers = [300, 900, 1800].map((delay) => window.setTimeout(ensurePreview, delay)); + const timers = [300, 900, 1800].map(delay => + window.setTimeout(ensurePreview, delay) + ); return () => { cancelled = true; - timers.forEach((timer) => window.clearTimeout(timer)); + timers.forEach(timer => window.clearTimeout(timer)); }; }, [bindLocalPreview, cameraActive, immersivePreview, runtimeRole]); @@ -882,67 +1162,88 @@ export default function LiveCamera() { return compositeCanvasRef.current; }, []); - const renderCompositeFrame = useCallback((landmarks?: Point[]) => { - const video = videoRef.current; - const compositeCanvas = ensureCompositeCanvas(); - if (!video || !compositeCanvas || video.videoWidth <= 0 || video.videoHeight <= 0) { - return; - } + const renderCompositeFrame = useCallback( + (landmarks?: Point[]) => { + const video = videoRef.current; + const compositeCanvas = ensureCompositeCanvas(); + if ( + !video || + !compositeCanvas || + video.videoWidth <= 0 || + video.videoHeight <= 0 + ) { + return; + } - if (compositeCanvas.width !== video.videoWidth || compositeCanvas.height !== video.videoHeight) { - compositeCanvas.width = video.videoWidth; - compositeCanvas.height = video.videoHeight; - } + if ( + compositeCanvas.width !== video.videoWidth || + compositeCanvas.height !== video.videoHeight + ) { + compositeCanvas.width = video.videoWidth; + compositeCanvas.height = video.videoHeight; + } - const ctx = compositeCanvas.getContext("2d"); - if (!ctx) return; + const ctx = compositeCanvas.getContext("2d"); + if (!ctx) return; - ctx.clearRect(0, 0, compositeCanvas.width, compositeCanvas.height); - ctx.drawImage(video, 0, 0, compositeCanvas.width, compositeCanvas.height); - renderLiveCameraOverlayToContext( - ctx, - compositeCanvas.width, - compositeCanvas.height, - landmarks, - avatarRenderRef.current, - { clear: false }, - ); - }, [ensureCompositeCanvas]); + ctx.clearRect(0, 0, compositeCanvas.width, compositeCanvas.height); + ctx.drawImage(video, 0, 0, compositeCanvas.width, compositeCanvas.height); + renderLiveCameraOverlayToContext( + ctx, + compositeCanvas.width, + compositeCanvas.height, + landmarks, + avatarRenderRef.current, + { clear: false } + ); + }, + [ensureCompositeCanvas] + ); - const queueArchivedVideoUpload = useCallback(async (blob: Blob, sequence: number, durationMs: number) => { - const format = recorderMimeTypeRef.current.includes("mp4") ? "mp4" : "webm"; - const title = `实时分析录像 ${formatDateTimeShanghai(new Date(), { - year: undefined, - second: undefined, - })} · 第 ${sequence} 段`; + const queueArchivedVideoUpload = useCallback( + async (blob: Blob, sequence: number, durationMs: number) => { + const format = recorderMimeTypeRef.current.includes("mp4") + ? "mp4" + : "webm"; + const title = `实时分析录像 ${formatDateTimeShanghai(new Date(), { + year: undefined, + second: undefined, + })} · 第 ${sequence} 段`; - recorderUploadQueueRef.current = recorderUploadQueueRef.current - .then(async () => { - const fileBase64 = await blobToBase64(blob); - const uploaded = await uploadMutation.mutateAsync({ - title, - format, - fileSize: blob.size, - duration: Math.max(1, Math.round(durationMs / 1000)), - exerciseType: "live_analysis", - fileBase64, + recorderUploadQueueRef.current = recorderUploadQueueRef.current + .then(async () => { + const fileBase64 = await blobToBase64(blob); + const uploaded = await uploadMutation.mutateAsync({ + title, + format, + fileSize: blob.size, + duration: Math.max(1, Math.round(durationMs / 1000)), + exerciseType: "live_analysis", + fileBase64, + }); + const nextVideo: ArchivedAnalysisVideo = { + videoId: uploaded.videoId, + url: uploaded.url, + sequence, + durationMs, + title, + }; + archivedVideosRef.current = [ + ...archivedVideosRef.current, + nextVideo, + ].sort((a, b) => a.sequence - b.sequence); + setArchivedVideoCount(archivedVideosRef.current.length); + }) + .catch((error: any) => { + toast.error( + `分析录像第 ${sequence} 段归档失败: ${error?.message || "未知错误"}` + ); }); - const nextVideo: ArchivedAnalysisVideo = { - videoId: uploaded.videoId, - url: uploaded.url, - sequence, - durationMs, - title, - }; - archivedVideosRef.current = [...archivedVideosRef.current, nextVideo].sort((a, b) => a.sequence - b.sequence); - setArchivedVideoCount(archivedVideosRef.current.length); - }) - .catch((error: any) => { - toast.error(`分析录像第 ${sequence} 段归档失败: ${error?.message || "未知错误"}`); - }); - return recorderUploadQueueRef.current; - }, [uploadMutation]); + return recorderUploadQueueRef.current; + }, + [uploadMutation] + ); const stopSessionRecorder = useCallback(async () => { const recorder = recorderRef.current; @@ -962,29 +1263,37 @@ export default function LiveCamera() { await recorderUploadQueueRef.current; }, []); - const buildRuntimeSnapshot = useCallback((phase?: RuntimeSnapshot["phase"]): RuntimeSnapshot => ({ - phase: phase ?? leaveStatusRef.current, - startedAt: sessionStartedAtRef.current || undefined, - durationMs: durationMsRef.current, - title: normalizedRuntimeTitle || `实时分析 ${ACTION_META[currentActionRef.current].label}`, - sessionMode: sessionModeRef.current, - qualityPreset, - facingMode: facing, - deviceKind: mobile ? "mobile" : "desktop", - avatarEnabled: avatarRenderRef.current.enabled, - avatarKey: avatarRenderRef.current.avatarKey, - avatarLabel: getAvatarPreset(avatarRenderRef.current.avatarKey)?.label || "猩猩", - updatedAt: Date.now(), - currentAction: currentActionRef.current, - rawAction: rawActionRef.current, - feedback: feedbackRef.current, - liveScore: liveScoreRef.current, - stabilityMeta: stabilityMetaRef.current, - visibleSegments: segmentsRef.current.filter((segment) => !segment.isUnknown).length, - unknownSegments: segmentsRef.current.filter((segment) => segment.isUnknown).length, - archivedVideoCount: archivedVideosRef.current.length, - recentSegments: segmentsRef.current.slice(-5), - }), [facing, mobile, normalizedRuntimeTitle, qualityPreset]); + const buildRuntimeSnapshot = useCallback( + (phase?: RuntimeSnapshot["phase"]): RuntimeSnapshot => ({ + phase: phase ?? leaveStatusRef.current, + startedAt: sessionStartedAtRef.current || undefined, + durationMs: durationMsRef.current, + title: + normalizedRuntimeTitle || + `实时分析 ${ACTION_META[currentActionRef.current].label}`, + sessionMode: sessionModeRef.current, + qualityPreset, + facingMode: facing, + deviceKind: mobile ? "mobile" : "desktop", + avatarEnabled: avatarRenderRef.current.enabled, + avatarKey: avatarRenderRef.current.avatarKey, + avatarLabel: + getAvatarPreset(avatarRenderRef.current.avatarKey)?.label || "猩猩", + updatedAt: Date.now(), + currentAction: currentActionRef.current, + rawAction: rawActionRef.current, + feedback: feedbackRef.current, + liveScore: liveScoreRef.current, + stabilityMeta: stabilityMetaRef.current, + visibleSegments: segmentsRef.current.filter(segment => !segment.isUnknown) + .length, + unknownSegments: segmentsRef.current.filter(segment => segment.isUnknown) + .length, + archivedVideoCount: archivedVideosRef.current.length, + recentSegments: segmentsRef.current.slice(-5), + }), + [facing, mobile, normalizedRuntimeTitle, qualityPreset] + ); const openSetupGuide = useCallback(async () => { const latest = await refreshRuntimeState(); @@ -996,120 +1305,105 @@ export default function LiveCamera() { setShowSetupGuide(true); }, [refreshRuntimeState]); - const uploadLiveFrame = useCallback(async (sessionId: string) => { - const compositeCanvas = ensureCompositeCanvas(); - if (!compositeCanvas || frameRelayInFlightRef.current) { - return; - } - - renderCompositeFrame(); - frameRelayInFlightRef.current = true; - try { - const blob = await new Promise((resolve) => { - compositeCanvas.toBlob(resolve, "image/jpeg", mobile ? 0.7 : 0.76); - }); - if (!blob) { - return; + const queueRelaySegmentUpload = useCallback( + (blob: Blob, sequence: number, durationMs: number) => { + const sessionId = broadcastSessionIdRef.current; + if (!sessionId || blob.size <= 0 || durationMs <= 0) { + return Promise.resolve(); } - await uploadMediaLiveFrame(sessionId, blob); - } finally { - frameRelayInFlightRef.current = false; - } - }, [ensureCompositeCanvas, mobile, renderCompositeFrame]); - const startFrameRelayLoop = useCallback((sessionId: string) => { - broadcastSessionIdRef.current = sessionId; - if (frameRelayTimerRef.current) { - window.clearInterval(frameRelayTimerRef.current); - frameRelayTimerRef.current = 0; - } + relayUploadQueueRef.current = relayUploadQueueRef.current + .then(async () => { + const response = await uploadMediaSegment( + sessionId, + Math.max(0, sequence - 1), + durationMs, + blob + ); + setViewerMediaSession(current => + current?.id === response.session.id ? response.session : current + ); + }) + .catch((error: any) => { + toast.error(`同步缓存上传失败: ${error?.message || "未知错误"}`); + }); - void uploadLiveFrame(sessionId); - frameRelayTimerRef.current = window.setInterval(() => { - void uploadLiveFrame(sessionId); - }, 900); - }, [uploadLiveFrame]); + return relayUploadQueueRef.current; + }, + [] + ); const closeBroadcastPeer = useCallback(() => { broadcastSessionIdRef.current = null; - if (frameRelayTimerRef.current) { - window.clearInterval(frameRelayTimerRef.current); - frameRelayTimerRef.current = 0; - } - frameRelayInFlightRef.current = false; }, []); - const closeViewerPeer = useCallback((options?: { clearFrameVersion?: boolean }) => { + const closeViewerPeer = useCallback(() => { if (viewerRetryTimerRef.current) { window.clearTimeout(viewerRetryTimerRef.current); viewerRetryTimerRef.current = 0; } - viewerSessionIdRef.current = null; - if (options?.clearFrameVersion) { - setViewerFrameVersion(0); - } + setViewerMediaSession(null); setViewerConnected(false); setViewerError(""); }, []); - const releaseRuntime = useCallback(async (phase: RuntimeSnapshot["phase"]) => { - if (!runtimeIdRef.current) return; + const releaseRuntime = useCallback( + async (phase: RuntimeSnapshot["phase"]) => { + if (!runtimeIdRef.current) return; + + try { + await runtimeReleaseMutation.mutateAsync({ + runtimeId: runtimeIdRef.current, + snapshot: buildRuntimeSnapshot(phase), + }); + } catch { + // Ignore runtime release errors and let the server-side stale timeout recover. + } finally { + runtimeIdRef.current = null; + broadcastSessionIdRef.current = null; + if (heartbeatTimerRef.current) { + window.clearInterval(heartbeatTimerRef.current); + heartbeatTimerRef.current = 0; + } + void runtimeQuery.refetch(); + } + }, + [buildRuntimeSnapshot, runtimeQuery, runtimeReleaseMutation] + ); + + const startRuntimeHeartbeatLoop = useCallback( + (mediaSessionId?: string | null) => { + if (!runtimeIdRef.current) return; + + if (typeof mediaSessionId === "string") { + broadcastSessionIdRef.current = mediaSessionId; + } - try { - await runtimeReleaseMutation.mutateAsync({ - runtimeId: runtimeIdRef.current, - snapshot: buildRuntimeSnapshot(phase), - }); - } catch { - // Ignore runtime release errors and let the server-side stale timeout recover. - } finally { - runtimeIdRef.current = null; - broadcastSessionIdRef.current = null; if (heartbeatTimerRef.current) { window.clearInterval(heartbeatTimerRef.current); heartbeatTimerRef.current = 0; } - void runtimeQuery.refetch(); - } - }, [buildRuntimeSnapshot, runtimeQuery, runtimeReleaseMutation]); - const startRuntimeHeartbeatLoop = useCallback((mediaSessionId?: string | null) => { - if (!runtimeIdRef.current) return; + const sendHeartbeat = () => { + if (!runtimeIdRef.current) return; + runtimeHeartbeatMutation.mutate({ + runtimeId: runtimeIdRef.current, + mediaSessionId: broadcastSessionIdRef.current, + snapshot: buildRuntimeSnapshot(), + }); + }; - if (typeof mediaSessionId === "string") { - broadcastSessionIdRef.current = mediaSessionId; - } - - if (heartbeatTimerRef.current) { - window.clearInterval(heartbeatTimerRef.current); - heartbeatTimerRef.current = 0; - } - - const sendHeartbeat = () => { - if (!runtimeIdRef.current) return; - runtimeHeartbeatMutation.mutate({ - runtimeId: runtimeIdRef.current, - mediaSessionId: broadcastSessionIdRef.current, - snapshot: buildRuntimeSnapshot(), - }); - }; - - sendHeartbeat(); - heartbeatTimerRef.current = window.setInterval(sendHeartbeat, 1000); - }, [buildRuntimeSnapshot, runtimeHeartbeatMutation]); + sendHeartbeat(); + heartbeatTimerRef.current = window.setInterval(sendHeartbeat, 1000); + }, + [buildRuntimeSnapshot, runtimeHeartbeatMutation] + ); const startBroadcastSession = useCallback(async () => { if (!user?.id) { throw new Error("当前用户信息未就绪"); } - const compositeCanvas = ensureCompositeCanvas(); - if (!compositeCanvas) { - throw new Error("当前浏览器不支持同步观看画面"); - } - - renderCompositeFrame(); - const sessionResponse = await createMediaSession({ userId: String(user.id), title: `实时分析同步 ${formatDateTimeShanghai(new Date(), { @@ -1121,24 +1415,23 @@ export default function LiveCamera() { qualityPreset, facingMode: facing, deviceKind: mobile ? "mobile" : "desktop", + purpose: "relay", }); const sessionId = sessionResponse.session.id; - startFrameRelayLoop(sessionId); + broadcastSessionIdRef.current = sessionId; return sessionId; - }, [ensureCompositeCanvas, facing, mobile, qualityPreset, renderCompositeFrame, startFrameRelayLoop, user?.id]); + }, [facing, mobile, qualityPreset, user?.id]); const startViewerStream = useCallback(async (mediaSessionId: string) => { - if (viewerSessionIdRef.current === mediaSessionId && viewerConnected) { - setViewerFrameVersion(Date.now()); - return; + const response = await getMediaSession(mediaSessionId); + setViewerMediaSession(response.session); + if (response.session.playback.previewUrl) { + setViewerError(""); + } else { + setViewerConnected(false); } - - closeViewerPeer(); - setViewerError(""); - viewerSessionIdRef.current = mediaSessionId; - setViewerFrameVersion(Date.now()); - }, [closeViewerPeer, viewerConnected]); + }, []); const stopCamera = useCallback(() => { if (animationRef.current) { @@ -1154,10 +1447,14 @@ export default function LiveCamera() { void stopSessionRecorder(); const localStream = streamRef.current; if (streamRef.current) { - streamRef.current.getTracks().forEach((track) => track.stop()); + streamRef.current.getTracks().forEach(track => track.stop()); streamRef.current = null; } - if (videoRef.current && localStream && videoRef.current.srcObject === localStream) { + if ( + videoRef.current && + localStream && + videoRef.current.srcObject === localStream + ) { videoRef.current.srcObject = null; } actionHistoryRef.current = []; @@ -1181,16 +1478,16 @@ export default function LiveCamera() { useEffect(() => { if (runtimeRole !== "viewer" || !runtimeSession?.mediaSessionId) { - closeViewerPeer({ - clearFrameVersion: !cameraActive, - }); + closeViewerPeer(); if (streamRef.current) { void bindLocalPreview(); } return; } - void startViewerStream(runtimeSession.mediaSessionId).catch((error: any) => { + const viewerMediaSessionId = runtimeSession.mediaSessionId; + + void startViewerStream(viewerMediaSessionId).catch((error: any) => { setViewerError(error?.message || "同步画面连接失败"); }); @@ -1199,8 +1496,10 @@ export default function LiveCamera() { viewerRetryTimerRef.current = 0; } viewerRetryTimerRef.current = window.setInterval(() => { - setViewerFrameVersion(Date.now()); - }, 900); + void startViewerStream(viewerMediaSessionId).catch((error: any) => { + setViewerError(error?.message || "同步画面刷新失败"); + }); + }, 5_000); return () => { if (viewerRetryTimerRef.current) { @@ -1210,7 +1509,6 @@ export default function LiveCamera() { }; }, [ bindLocalPreview, - cameraActive, closeViewerPeer, runtimeRole, runtimeSession?.mediaSessionId, @@ -1225,26 +1523,34 @@ export default function LiveCamera() { }; }, [closeBroadcastPeer, closeViewerPeer, stopCamera]); - const syncZoomState = useCallback(async (preferredZoom?: number, providedTrack?: MediaStreamTrack | null) => { - const track = providedTrack || streamRef.current?.getVideoTracks()[0] || null; - if (!track) { - zoomTargetRef.current = 1; - setZoomState(readTrackZoomState(null)); - return; - } - - let nextState = readTrackZoomState(track); - if (nextState.supported && preferredZoom != null && Math.abs(preferredZoom - nextState.current) > nextState.step / 2) { - try { - nextState = await applyTrackZoom(track, preferredZoom); - } catch { - nextState = readTrackZoomState(track); + const syncZoomState = useCallback( + async (preferredZoom?: number, providedTrack?: MediaStreamTrack | null) => { + const track = + providedTrack || streamRef.current?.getVideoTracks()[0] || null; + if (!track) { + zoomTargetRef.current = 1; + setZoomState(readTrackZoomState(null)); + return; } - } - zoomTargetRef.current = nextState.current; - setZoomState(nextState); - }, []); + let nextState = readTrackZoomState(track); + if ( + nextState.supported && + preferredZoom != null && + Math.abs(preferredZoom - nextState.current) > nextState.step / 2 + ) { + try { + nextState = await applyTrackZoom(track, preferredZoom); + } catch { + nextState = readTrackZoomState(track); + } + } + + zoomTargetRef.current = nextState.current; + setZoomState(nextState); + }, + [] + ); const updateZoom = useCallback(async (nextZoom: number) => { const track = streamRef.current?.getVideoTracks()[0] || null; @@ -1259,63 +1565,85 @@ export default function LiveCamera() { } }, []); - const stepZoom = useCallback((direction: -1 | 1) => { - if (!zoomState.supported) return; - const nextZoom = clamp(zoomState.current + zoomState.step * direction, zoomState.min, zoomState.max); - void updateZoom(nextZoom); - }, [updateZoom, zoomState]); + const stepZoom = useCallback( + (direction: -1 | 1) => { + if (!zoomState.supported) return; + const nextZoom = clamp( + zoomState.current + zoomState.step * direction, + zoomState.min, + zoomState.max + ); + void updateZoom(nextZoom); + }, + [updateZoom, zoomState] + ); - const startCamera = useCallback(async ( - nextFacing: CameraFacing = facing, - preferredZoom = zoomTargetRef.current, - preset: CameraQualityPreset = qualityPreset, - ) => { - const latest = await refreshRuntimeState(); - if (latest.role === "viewer") { - toast.error("当前账号已有其他设备正在实时分析，请切换到同步观看模式"); - return; - } - try { - if (streamRef.current) { - streamRef.current.getTracks().forEach((track) => track.stop()); + const startCamera = useCallback( + async ( + nextFacing: CameraFacing = facing, + preferredZoom = zoomTargetRef.current, + preset: CameraQualityPreset = qualityPreset + ) => { + const latest = await refreshRuntimeState(); + if (latest.role === "viewer") { + toast.error("当前账号已有其他设备正在实时分析，请切换到同步观看模式"); + return; } - const { stream, appliedFacingMode, usedFallback } = await requestCameraStream({ - facingMode: nextFacing, - isMobile: mobile, - preset, - }); - streamRef.current = stream; - closeViewerPeer(); - if (appliedFacingMode !== nextFacing) { - setFacing(appliedFacingMode); + try { + if (streamRef.current) { + streamRef.current.getTracks().forEach(track => track.stop()); + } + const { stream, appliedFacingMode, usedFallback } = + await requestCameraStream({ + facingMode: nextFacing, + isMobile: mobile, + preset, + }); + streamRef.current = stream; + closeViewerPeer(); + if (appliedFacingMode !== nextFacing) { + setFacing(appliedFacingMode); + } + await bindLocalPreview(stream); + setCameraActive(true); + await syncZoomState(preferredZoom, stream.getVideoTracks()[0] || null); + if (usedFallback) { + toast.info("当前设备已自动切换到兼容摄像头模式"); + } + toast.success("摄像头已启动"); + } catch (error: any) { + toast.error(`摄像头启动失败: ${error?.message || "未知错误"}`); } - await bindLocalPreview(stream); - setCameraActive(true); - await syncZoomState(preferredZoom, stream.getVideoTracks()[0] || null); - if (usedFallback) { - toast.info("当前设备已自动切换到兼容摄像头模式"); - } - toast.success("摄像头已启动"); - } catch (error: any) { - toast.error(`摄像头启动失败: ${error?.message || "未知错误"}`); - } - }, [bindLocalPreview, closeViewerPeer, facing, mobile, qualityPreset, refreshRuntimeState, syncZoomState]); + }, + [ + bindLocalPreview, + closeViewerPeer, + facing, + mobile, + qualityPreset, + refreshRuntimeState, + syncZoomState, + ] + ); const switchCamera = useCallback(async () => { const nextFacing: CameraFacing = facing === "user" ? "environment" : "user"; setFacing(nextFacing); if (!cameraActive) return; stopCamera(); - await new Promise((resolve) => setTimeout(resolve, 250)); + await new Promise(resolve => setTimeout(resolve, 250)); await startCamera(nextFacing, zoomTargetRef.current); }, [cameraActive, facing, startCamera, stopCamera]); - const handleQualityPresetChange = useCallback(async (nextPreset: CameraQualityPreset) => { - setQualityPreset(nextPreset); - if (cameraActive && !analyzing && !saving) { - await startCamera(facing, zoomTargetRef.current, nextPreset); - } - }, [analyzing, cameraActive, facing, saving, startCamera]); + const handleQualityPresetChange = useCallback( + async (nextPreset: CameraQualityPreset) => { + setQualityPreset(nextPreset); + if (cameraActive && !analyzing && !saving) { + await startCamera(facing, zoomTargetRef.current, nextPreset); + } + }, + [analyzing, cameraActive, facing, saving, startCamera] + ); const flushSegment = useCallback((segment: ActionSegment | null) => { if (!segment || segment.durationMs < MIN_SEGMENT_MS) { @@ -1332,98 +1660,137 @@ export default function LiveCamera() { setSegments(segmentsRef.current); }, []); - const appendFrameToSegment = useCallback((frame: AnalyzedFrame, elapsedMs: number) => { - const current = currentSegmentRef.current; - if (!current) { + const appendFrameToSegment = useCallback( + (frame: AnalyzedFrame, elapsedMs: number) => { + const current = currentSegmentRef.current; + if (!current) { + currentSegmentRef.current = createSegment( + frame.action, + elapsedMs, + frame + ); + return; + } + + const sameAction = current.actionType === frame.action; + const gap = elapsedMs - current.endMs; + const nextDuration = elapsedMs - current.startMs; + + if (sameAction && gap <= MERGE_GAP_MS && nextDuration <= SEGMENT_MAX_MS) { + const nextFrameCount = current.frameCount + 1; + current.endMs = elapsedMs; + current.durationMs = current.endMs - current.startMs; + current.frameCount = nextFrameCount; + current.confidenceAvg = + (current.confidenceAvg * (nextFrameCount - 1) + frame.confidence) / + nextFrameCount; + current.score = + (current.score * (nextFrameCount - 1) + frame.score.overall) / + nextFrameCount; + current.peakScore = Math.max(current.peakScore, frame.score.overall); + current.issueSummary = Array.from( + new Set([...current.issueSummary, ...frame.feedback]) + ).slice(0, 4); + current.keyFrames = [...current.keyFrames.slice(-3), elapsedMs]; + return; + } + + flushSegment(current); currentSegmentRef.current = createSegment(frame.action, elapsedMs, frame); - return; - } + }, + [flushSegment] + ); - const sameAction = current.actionType === frame.action; - const gap = elapsedMs - current.endMs; - const nextDuration = elapsedMs - current.startMs; - - if (sameAction && gap <= MERGE_GAP_MS && nextDuration <= SEGMENT_MAX_MS) { - const nextFrameCount = current.frameCount + 1; - current.endMs = elapsedMs; - current.durationMs = current.endMs - current.startMs; - current.frameCount = nextFrameCount; - current.confidenceAvg = ((current.confidenceAvg * (nextFrameCount - 1)) + frame.confidence) / nextFrameCount; - current.score = ((current.score * (nextFrameCount - 1)) + frame.score.overall) / nextFrameCount; - current.peakScore = Math.max(current.peakScore, frame.score.overall); - current.issueSummary = Array.from(new Set([...current.issueSummary, ...frame.feedback])).slice(0, 4); - current.keyFrames = [...current.keyFrames.slice(-3), elapsedMs]; - return; - } - - flushSegment(current); - currentSegmentRef.current = createSegment(frame.action, elapsedMs, frame); - }, [flushSegment]); - - const startSessionRecorder = useCallback(function startSessionRecorderInternal() { - if (typeof MediaRecorder === "undefined") { - recorderRef.current = null; - recorderStopPromiseRef.current = Promise.resolve(); - return; - } - - const compositeCanvas = ensureCompositeCanvas(); - if (!compositeCanvas || typeof compositeCanvas.captureStream !== "function") { - recorderRef.current = null; - recorderStopPromiseRef.current = Promise.resolve(); - return; - } - - renderCompositeFrame(); - recorderChunksRef.current = []; - const mimeType = pickRecorderMimeType(); - recorderMimeTypeRef.current = mimeType; - if (!recorderStreamRef.current) { - recorderStreamRef.current = compositeCanvas.captureStream(mobile ? 24 : 30); - } - const recorder = new MediaRecorder(recorderStreamRef.current, { - mimeType, - videoBitsPerSecond: getLiveAnalysisBitrate(qualityPreset, mobile), - }); - recorderRef.current = recorder; - const sequence = recorderSequenceRef.current + 1; - recorderSequenceRef.current = sequence; - recorderSegmentStartedAtRef.current = Date.now(); - - recorder.ondataavailable = (event) => { - if (event.data && event.data.size > 0) { - recorderChunksRef.current.push(event.data); - } - }; - - recorderStopPromiseRef.current = new Promise((resolve) => { - recorder.onstop = () => { - const durationMs = Math.max(0, Date.now() - recorderSegmentStartedAtRef.current); - const type = recorderMimeTypeRef.current.includes("mp4") ? "video/mp4" : "video/webm"; - const blob = recorderChunksRef.current.length > 0 ? new Blob(recorderChunksRef.current, { type }) : null; - recorderChunksRef.current = []; + const startSessionRecorder = useCallback( + function startSessionRecorderInternal() { + if (typeof MediaRecorder === "undefined") { recorderRef.current = null; - recorderStopPromiseRef.current = null; - if (blob && blob.size > 0 && durationMs > 0) { - void queueArchivedVideoUpload(blob, sequence, durationMs); - } - if (analyzingRef.current) { - startSessionRecorderInternal(); - } else if (recorderStreamRef.current) { - recorderStreamRef.current.getTracks().forEach((track) => track.stop()); - recorderStreamRef.current = null; - } - resolve(); - }; - }); - - recorder.start(); - recorderRotateTimerRef.current = window.setTimeout(() => { - if (recorder.state === "recording") { - recorder.stop(); + recorderStopPromiseRef.current = Promise.resolve(); + return; } - }, ANALYSIS_RECORDING_SEGMENT_MS); - }, [ensureCompositeCanvas, mobile, qualityPreset, queueArchivedVideoUpload, renderCompositeFrame]); + + const compositeCanvas = ensureCompositeCanvas(); + if ( + !compositeCanvas || + typeof compositeCanvas.captureStream !== "function" + ) { + recorderRef.current = null; + recorderStopPromiseRef.current = Promise.resolve(); + return; + } + + renderCompositeFrame(); + recorderChunksRef.current = []; + const mimeType = pickRecorderMimeType(); + recorderMimeTypeRef.current = mimeType; + if (!recorderStreamRef.current) { + recorderStreamRef.current = compositeCanvas.captureStream( + mobile ? 24 : 30 + ); + } + const recorder = new MediaRecorder(recorderStreamRef.current, { + mimeType, + videoBitsPerSecond: getLiveAnalysisBitrate(qualityPreset, mobile), + }); + recorderRef.current = recorder; + const sequence = recorderSequenceRef.current + 1; + recorderSequenceRef.current = sequence; + recorderSegmentStartedAtRef.current = Date.now(); + + recorder.ondataavailable = event => { + if (event.data && event.data.size > 0) { + recorderChunksRef.current.push(event.data); + } + }; + + recorderStopPromiseRef.current = new Promise(resolve => { + recorder.onstop = () => { + const durationMs = Math.max( + 0, + Date.now() - recorderSegmentStartedAtRef.current + ); + const type = recorderMimeTypeRef.current.includes("mp4") + ? "video/mp4" + : "video/webm"; + const blob = + recorderChunksRef.current.length > 0 + ? new Blob(recorderChunksRef.current, { type }) + : null; + recorderChunksRef.current = []; + recorderRef.current = null; + recorderStopPromiseRef.current = null; + if (blob && blob.size > 0 && durationMs > 0) { + void queueRelaySegmentUpload(blob, sequence, durationMs); + void queueArchivedVideoUpload(blob, sequence, durationMs); + } + if (analyzingRef.current) { + startSessionRecorderInternal(); + } else if (recorderStreamRef.current) { + recorderStreamRef.current + .getTracks() + .forEach(track => track.stop()); + recorderStreamRef.current = null; + } + resolve(); + }; + }); + + recorder.start(); + recorderRotateTimerRef.current = window.setTimeout(() => { + if (recorder.state === "recording") { + recorder.stop(); + } + }, ANALYSIS_RECORDING_SEGMENT_MS); + }, + [ + ensureCompositeCanvas, + mobile, + qualityPreset, + queueArchivedVideoUpload, + queueRelaySegmentUpload, + renderCompositeFrame, + ] + ); const persistSession = useCallback(async () => { const endedAt = Date.now(); @@ -1438,37 +1805,73 @@ export default function LiveCamera() { const scoreSamples = frameSamplesRef.current; const finalSegments = [...segmentsRef.current]; - const segmentDurations = finalSegments.reduce>((acc, segment) => { - acc[segment.actionType] = (acc[segment.actionType] || 0) + segment.durationMs; - return acc; - }, { - forehand: 0, - backhand: 0, - serve: 0, - volley: 0, - overhead: 0, - slice: 0, - lob: 0, - unknown: 0, - }); - const dominantAction = (Object.entries(segmentDurations).sort((a, b) => b[1] - a[1])[0]?.[0] || "unknown") as ActionType; - const effectiveSegments = finalSegments.filter((segment) => !segment.isUnknown); + const segmentDurations = finalSegments.reduce>( + (acc, segment) => { + acc[segment.actionType] = + (acc[segment.actionType] || 0) + segment.durationMs; + return acc; + }, + { + forehand: 0, + backhand: 0, + serve: 0, + volley: 0, + overhead: 0, + slice: 0, + lob: 0, + unknown: 0, + } + ); + const dominantAction = (Object.entries(segmentDurations).sort( + (a, b) => b[1] - a[1] + )[0]?.[0] || "unknown") as ActionType; + const effectiveSegments = finalSegments.filter( + segment => !segment.isUnknown + ); const unknownCount = finalSegments.length - effectiveSegments.length; - const averageScore = scoreSamples.length > 0 - ? scoreSamples.reduce((sum, item) => sum + item.overall, 0) / scoreSamples.length - : liveScore?.overall || 0; - const averagePosture = scoreSamples.length > 0 ? scoreSamples.reduce((sum, item) => sum + item.posture, 0) / scoreSamples.length : liveScore?.posture || 0; - const averageBalance = scoreSamples.length > 0 ? scoreSamples.reduce((sum, item) => sum + item.balance, 0) / scoreSamples.length : liveScore?.balance || 0; - const averageTechnique = scoreSamples.length > 0 ? scoreSamples.reduce((sum, item) => sum + item.technique, 0) / scoreSamples.length : liveScore?.technique || 0; - const averageFootwork = scoreSamples.length > 0 ? scoreSamples.reduce((sum, item) => sum + item.footwork, 0) / scoreSamples.length : liveScore?.footwork || 0; - const averageConsistency = scoreSamples.length > 0 ? scoreSamples.reduce((sum, item) => sum + item.consistency, 0) / scoreSamples.length : liveScore?.consistency || 0; - const sessionFeedback = Array.from(new Set(finalSegments.flatMap((segment) => segment.issueSummary))).slice(0, 5); - const averageRawVolatility = volatilitySamplesRef.current.length > 0 - ? volatilitySamplesRef.current.reduce((sum, value) => sum + value, 0) / volatilitySamplesRef.current.length - : 0; + const averageScore = + scoreSamples.length > 0 + ? scoreSamples.reduce((sum, item) => sum + item.overall, 0) / + scoreSamples.length + : liveScore?.overall || 0; + const averagePosture = + scoreSamples.length > 0 + ? scoreSamples.reduce((sum, item) => sum + item.posture, 0) / + scoreSamples.length + : liveScore?.posture || 0; + const averageBalance = + scoreSamples.length > 0 + ? scoreSamples.reduce((sum, item) => sum + item.balance, 0) / + scoreSamples.length + : liveScore?.balance || 0; + const averageTechnique = + scoreSamples.length > 0 + ? scoreSamples.reduce((sum, item) => sum + item.technique, 0) / + scoreSamples.length + : liveScore?.technique || 0; + const averageFootwork = + scoreSamples.length > 0 + ? scoreSamples.reduce((sum, item) => sum + item.footwork, 0) / + scoreSamples.length + : liveScore?.footwork || 0; + const averageConsistency = + scoreSamples.length > 0 + ? scoreSamples.reduce((sum, item) => sum + item.consistency, 0) / + scoreSamples.length + : liveScore?.consistency || 0; + const sessionFeedback = Array.from( + new Set(finalSegments.flatMap(segment => segment.issueSummary)) + ).slice(0, 5); + const averageRawVolatility = + volatilitySamplesRef.current.length > 0 + ? volatilitySamplesRef.current.reduce((sum, value) => sum + value, 0) / + volatilitySamplesRef.current.length + : 0; const avatarState = avatarRenderRef.current; await stopSessionRecorder(); - const archivedVideos = [...archivedVideosRef.current].sort((a, b) => a.sequence - b.sequence); + const archivedVideos = [...archivedVideosRef.current].sort( + (a, b) => a.sequence - b.sequence + ); const primaryArchivedVideo = archivedVideos[0] ?? null; await saveLiveSessionMutation.mutateAsync({ @@ -1492,7 +1895,12 @@ export default function LiveCamera() { metrics: { actionDurations: segmentDurations, stabilizedActionDurations: segmentDurations, - averageConfidence: Math.round((scoreSamples.reduce((sum, item) => sum + item.confidence, 0) / Math.max(1, scoreSamples.length)) * 10) / 10, + averageConfidence: + Math.round( + (scoreSamples.reduce((sum, item) => sum + item.confidence, 0) / + Math.max(1, scoreSamples.length)) * + 10 + ) / 10, sampleCount: scoreSamples.length, stableWindowFrames: ACTION_WINDOW_FRAMES, actionSwitchCount: stableActionStateRef.current.switchCount, @@ -1504,7 +1912,7 @@ export default function LiveCamera() { archivedVideos, mobile, }, - segments: finalSegments.map((segment) => ({ + segments: finalSegments.map(segment => ({ actionType: segment.actionType, isUnknown: segment.isUnknown, startMs: segment.startMs, @@ -1521,7 +1929,14 @@ export default function LiveCamera() { videoId: primaryArchivedVideo?.videoId, videoUrl: primaryArchivedVideo?.url, }); - }, [flushSegment, liveScore, mobile, saveLiveSessionMutation, sessionMode, stopSessionRecorder]); + }, [ + flushSegment, + liveScore, + mobile, + saveLiveSessionMutation, + sessionMode, + stopSessionRecorder, + ]); const startAnalysis = useCallback(async () => { const latest = await refreshRuntimeState(); @@ -1544,7 +1959,9 @@ export default function LiveCamera() { if (runtime.role === "viewer") { runtimeIdRef.current = null; - toast.error("同一账号已有其他设备正在实时分析，本机已切换为同步观看模式"); + toast.error( + "同一账号已有其他设备正在实时分析，本机已切换为同步观看模式" + ); await runtimeQuery.refetch(); return; } @@ -1585,9 +2002,12 @@ export default function LiveCamera() { } ).__TEST_MEDIAPIPE_FACTORY__; - const { Pose } = testFactory ? await testFactory() : await import("@mediapipe/pose"); + const { Pose } = testFactory + ? await testFactory() + : await import("@mediapipe/pose"); const pose = new Pose({ - locateFile: (file: string) => `https://cdn.jsdelivr.net/npm/@mediapipe/pose/${file}`, + locateFile: (file: string) => + `https://cdn.jsdelivr.net/npm/@mediapipe/pose/${file}`, }); poseRef.current = pose; @@ -1609,12 +2029,20 @@ export default function LiveCamera() { canvas.height = video.videoHeight; } - drawLiveCameraOverlay(canvas, results.poseLandmarks, avatarRenderRef.current); + drawLiveCameraOverlay( + canvas, + results.poseLandmarks, + avatarRenderRef.current + ); renderCompositeFrame(results.poseLandmarks); if (!results.poseLandmarks) return; const frameTimestamp = performance.now(); - const analyzed = analyzePoseFrame(results.poseLandmarks, trackingRef.current, frameTimestamp); + const analyzed = analyzePoseFrame( + results.poseLandmarks, + trackingRef.current, + frameTimestamp + ); const nextStabilityMeta = stabilizeActionStream( { action: analyzed.action, @@ -1622,17 +2050,28 @@ export default function LiveCamera() { timestamp: frameTimestamp, }, actionHistoryRef.current, - stableActionStateRef.current, + stableActionStateRef.current ); const elapsedMs = Date.now() - sessionStartedAtRef.current; - const stabilityLabel = nextStabilityMeta.pendingAction ?? nextStabilityMeta.windowAction; - const stabilityFeedback = nextStabilityMeta.pending && stabilityLabel !== "unknown" - ? [`正在确认 ${ACTION_META[stabilityLabel].label}，需要持续约 0.7 秒后再切换。`, ...analyzed.feedback] - : nextStabilityMeta.stableAction === "unknown" - ? ["系统正在积累 24 帧动作窗口，当前先作为观察片段处理。", ...analyzed.feedback] - : analyzed.action !== nextStabilityMeta.stableAction - ? [`原始候选为 ${ACTION_META[analyzed.action].label}，当前保持 ${ACTION_META[nextStabilityMeta.stableAction].label}。`, ...analyzed.feedback] - : analyzed.feedback; + const stabilityLabel = + nextStabilityMeta.pendingAction ?? nextStabilityMeta.windowAction; + const stabilityFeedback = + nextStabilityMeta.pending && stabilityLabel !== "unknown" + ? [ + `正在确认 ${ACTION_META[stabilityLabel].label}，需要持续约 0.7 秒后再切换。`, + ...analyzed.feedback, + ] + : nextStabilityMeta.stableAction === "unknown" + ? [ + "系统正在积累 24 帧动作窗口，当前先作为观察片段处理。", + ...analyzed.feedback, + ] + : analyzed.action !== nextStabilityMeta.stableAction + ? [ + `原始候选为 ${ACTION_META[analyzed.action].label}，当前保持 ${ACTION_META[nextStabilityMeta.stableAction].label}。`, + ...analyzed.feedback, + ] + : analyzed.feedback; const displayedScore: PoseScore = { ...analyzed.score, confidence: Math.round(nextStabilityMeta.stableConfidence * 100), @@ -1656,7 +2095,8 @@ export default function LiveCamera() { }); const processFrame = async () => { - if (!analyzingRef.current || !videoRef.current || !poseRef.current) return; + if (!analyzingRef.current || !videoRef.current || !poseRef.current) + return; if (videoRef.current.readyState >= 2 || testFactory) { await poseRef.current.send({ image: videoRef.current }); } @@ -1711,7 +2151,9 @@ export default function LiveCamera() { await persistSession(); setLeaveStatus("safe"); releasePhase = "safe"; - toast.success(`实时分析已保存，并同步写入训练记录${archivedVideosRef.current.length > 0 ? `；已归档 ${archivedVideosRef.current.length} 段分析录像` : ""}`); + toast.success( + `实时分析已保存，并同步写入训练记录${archivedVideosRef.current.length > 0 ? `；已归档 ${archivedVideosRef.current.length} 段分析录像` : ""}` + ); await liveSessionsQuery.refetch(); } catch (error: any) { setLeaveStatus("failed"); @@ -1750,69 +2192,113 @@ export default function LiveCamera() { await startCamera(facing, zoomTargetRef.current, qualityPreset); }, [facing, qualityPreset, refreshRuntimeState, startCamera]); - const displayLeaveStatus = runtimeRole === "viewer" ? (runtimeSnapshot?.phase ?? "idle") : leaveStatus; - const displayAction = runtimeRole === "viewer" ? (runtimeSnapshot?.currentAction ?? "unknown") : currentAction; - const displayRawAction = runtimeRole === "viewer" ? (runtimeSnapshot?.rawAction ?? "unknown") : rawAction; - const displayScore = runtimeRole === "viewer" ? (runtimeSnapshot?.liveScore ?? null) : liveScore; - const displayFeedback = runtimeRole === "viewer" ? (runtimeSnapshot?.feedback ?? []) : feedback; - const displayDurationMs = runtimeRole === "viewer" ? (runtimeSnapshot?.durationMs ?? 0) : durationMs; - const displayStabilityMeta = runtimeRole === "viewer" - ? { - ...createEmptyStabilizedActionMeta(), - ...runtimeSnapshot?.stabilityMeta, - } - : stabilityMeta; - const displaySessionMode = runtimeRole === "viewer" - ? (runtimeSnapshot?.sessionMode ?? runtimeSession?.sessionMode ?? sessionMode) - : sessionMode; - const displayQualityPreset = runtimeRole === "viewer" - ? (runtimeSnapshot?.qualityPreset ?? qualityPreset) - : qualityPreset; - const displayFacing = runtimeRole === "viewer" - ? (runtimeSnapshot?.facingMode ?? facing) - : facing; - const displayDeviceKind = runtimeRole === "viewer" - ? (runtimeSnapshot?.deviceKind ?? (mobile ? "mobile" : "desktop")) - : (mobile ? "mobile" : "desktop"); - const displayAvatarEnabled = runtimeRole === "viewer" - ? Boolean(runtimeSnapshot?.avatarEnabled) - : avatarEnabled; - const displayAvatarKey = runtimeRole === "viewer" - ? ((runtimeSnapshot?.avatarKey as AvatarKey | undefined) ?? resolvedAvatarKey) - : resolvedAvatarKey; + const displayLeaveStatus = + runtimeRole === "viewer" ? (runtimeSnapshot?.phase ?? "idle") : leaveStatus; + const displayAction = + runtimeRole === "viewer" + ? (runtimeSnapshot?.currentAction ?? "unknown") + : currentAction; + const displayRawAction = + runtimeRole === "viewer" + ? (runtimeSnapshot?.rawAction ?? "unknown") + : rawAction; + const displayScore = + runtimeRole === "viewer" ? (runtimeSnapshot?.liveScore ?? null) : liveScore; + const displayFeedback = + runtimeRole === "viewer" ? (runtimeSnapshot?.feedback ?? []) : feedback; + const displayDurationMs = + runtimeRole === "viewer" ? (runtimeSnapshot?.durationMs ?? 0) : durationMs; + const displayStabilityMeta = + runtimeRole === "viewer" + ? { + ...createEmptyStabilizedActionMeta(), + ...runtimeSnapshot?.stabilityMeta, + } + : stabilityMeta; + const displaySessionMode = + runtimeRole === "viewer" + ? (runtimeSnapshot?.sessionMode ?? + runtimeSession?.sessionMode ?? + sessionMode) + : sessionMode; + const displayQualityPreset = + runtimeRole === "viewer" + ? (runtimeSnapshot?.qualityPreset ?? qualityPreset) + : qualityPreset; + const displayFacing = + runtimeRole === "viewer" ? (runtimeSnapshot?.facingMode ?? facing) : facing; + const displayDeviceKind = + runtimeRole === "viewer" + ? (runtimeSnapshot?.deviceKind ?? (mobile ? "mobile" : "desktop")) + : mobile + ? "mobile" + : "desktop"; + const displayAvatarEnabled = + runtimeRole === "viewer" + ? Boolean(runtimeSnapshot?.avatarEnabled) + : avatarEnabled; + const displayAvatarKey = + runtimeRole === "viewer" + ? ((runtimeSnapshot?.avatarKey as AvatarKey | undefined) ?? + resolvedAvatarKey) + : resolvedAvatarKey; const displayAvatarPreset = getAvatarPreset(displayAvatarKey); - const displayAvatarLabel = runtimeRole === "viewer" - ? (runtimeSnapshot?.avatarLabel ?? displayAvatarPreset?.label ?? "猩猩") - : (displayAvatarPreset?.label || "猩猩"); - const runtimeSyncDelayMs = runtimeRole === "viewer" ? getRuntimeSyncDelayMs(runtimeSession?.lastHeartbeatAt) : null; - const runtimeSyncLabel = runtimeRole === "viewer" ? formatRuntimeSyncDelay(runtimeSyncDelayMs) : ""; - const displayRuntimeTitle = runtimeRole === "viewer" - ? (normalizedSnapshotTitle || normalizedRuntimeTitle || "其他设备实时分析") - : (normalizedRuntimeTitle || `实时分析 ${ACTION_META[currentAction].label}`); - const viewerFrameSrc = runtimeRole === "viewer" && runtimeSession?.mediaSessionId - ? getMediaAssetUrl(`/assets/sessions/${runtimeSession.mediaSessionId}/live-frame.jpg?ts=${viewerFrameVersion || runtimeSnapshot?.updatedAt || Date.now()}`) - : ""; - const hasVideoFeed = runtimeRole === "viewer" ? viewerConnected : cameraActive; + const displayAvatarLabel = + runtimeRole === "viewer" + ? (runtimeSnapshot?.avatarLabel ?? displayAvatarPreset?.label ?? "猩猩") + : displayAvatarPreset?.label || "猩猩"; + const runtimeSyncDelayMs = + runtimeRole === "viewer" + ? getRuntimeSyncDelayMs(runtimeSession?.lastHeartbeatAt) + : null; + const runtimeSyncLabel = + runtimeRole === "viewer" ? formatRuntimeSyncDelay(runtimeSyncDelayMs) : ""; + const displayRuntimeTitle = + runtimeRole === "viewer" + ? normalizedSnapshotTitle || normalizedRuntimeTitle || "其他设备实时分析" + : normalizedRuntimeTitle || + `实时分析 ${ACTION_META[currentAction].label}`; + const viewerPlaybackSrc = + runtimeRole === "viewer" && viewerMediaSession?.playback.previewUrl + ? getMediaAssetUrl( + `${viewerMediaSession.playback.previewUrl}?ts=${viewerMediaSession.previewUpdatedAt || runtimeSnapshot?.updatedAt || Date.now()}` + ) + : ""; + const viewerBufferReady = + runtimeRole === "viewer" && + Boolean(viewerMediaSession?.playback.previewUrl); + const hasVideoFeed = + runtimeRole === "viewer" ? viewerConnected : cameraActive; const heroAction = ACTION_META[displayAction]; const rawActionMeta = ACTION_META[displayRawAction]; - const pendingActionMeta = displayStabilityMeta.pendingAction ? ACTION_META[displayStabilityMeta.pendingAction] : null; + const pendingActionMeta = displayStabilityMeta.pendingAction + ? ACTION_META[displayStabilityMeta.pendingAction] + : null; const resolvedAvatarPreset = getAvatarPreset(resolvedAvatarKey); const resolvedAvatarLabel = resolvedAvatarPreset?.label || "猩猩"; - const animalAvatarPresets = AVATAR_PRESETS.filter((preset) => preset.category === "animal"); - const fullBodyAvatarPresets = AVATAR_PRESETS.filter((preset) => preset.category === "full-body-3d"); - const previewTitle = runtimeRole === "viewer" - ? viewerConnected - ? `${runtimeSyncLabel} · 服务端同步中` - : "正在获取服务端同步画面" - : analyzing - ? displayStabilityMeta.pending && pendingActionMeta - ? `${pendingActionMeta.label} 切换确认中` - : `${heroAction.label} 识别中` - : cameraActive - ? "准备开始实时分析" - : "摄像头待启动"; + const animalAvatarPresets = AVATAR_PRESETS.filter( + preset => preset.category === "animal" + ); + const fullBodyAvatarPresets = AVATAR_PRESETS.filter( + preset => preset.category === "full-body-3d" + ); + const previewTitle = + runtimeRole === "viewer" + ? viewerConnected + ? `${runtimeSyncLabel} · 服务端缓存同步中` + : viewerBufferReady + ? "正在加载最近 60 秒缓存" + : "正在缓冲最近 60 秒视频" + : analyzing + ? displayStabilityMeta.pending && pendingActionMeta + ? `${pendingActionMeta.label} 切换确认中` + : `${heroAction.label} 识别中` + : cameraActive + ? "准备开始实时分析" + : "摄像头待启动"; - const viewerModeLabel = normalizedSnapshotTitle || normalizedRuntimeTitle || "其他设备正在实时分析"; + const viewerModeLabel = + normalizedSnapshotTitle || normalizedRuntimeTitle || "其他设备正在实时分析"; const renderPrimaryActions = (rail = false) => { const buttonClass = rail @@ -1827,9 +2313,13 @@ export default function LiveCamera() { className={buttonClass} onClick={() => { if (runtimeSession?.mediaSessionId) { - void startViewerStream(runtimeSession.mediaSessionId).catch((error: any) => { - toast.error(`同步观看连接失败: ${error?.message || "未知错误"}`); - }); + void startViewerStream(runtimeSession.mediaSessionId).catch( + (error: any) => { + toast.error( + `同步观看连接失败: ${error?.message || "未知错误"}` + ); + } + ); } }} disabled={!runtimeSession?.mediaSessionId} @@ -1863,7 +2353,11 @@ export default function LiveCamera() { return ( <> {hasMultipleCameras ? ( - @@ -1879,13 +2373,23 @@ export default function LiveCamera() { {!rail && "开始分析"} ) : ( - )} {!rail ? ( - @@ -1907,8 +2411,12 @@ export default function LiveCamera() {
-
焦距
-
{zoomState.supported ? `${zoomState.current.toFixed(1)}x` : "自动"}
+
+ 焦距 +
+
+ {zoomState.supported ? `${zoomState.current.toFixed(1)}x` : "自动"} +
{setupStep > 0 ? ( - + ) : null} {setupStep < SETUP_STEPS.length - 1 ? ( - + ) : (
-

{displayRuntimeTitle}

+

+ {displayRuntimeTitle} +

{runtimeRole === "viewer" - ? `当前正在通过服务端中转同步 ${displayDeviceKind === "mobile" ? "移动端" : "桌面端"} ${displayFacing === "environment" ? "后置/主摄视角" : "前置视角"} 画面。同步画面、动作、评分、最近区间、虚拟形象和会话状态会自动跟随持有端刷新，允许少量网络延迟。` + ? `当前正在通过服务端中转同步 ${displayDeviceKind === "mobile" ? "移动端" : "桌面端"} ${displayFacing === "environment" ? "后置/主摄视角" : "前置视角"} 画面。持有端会持续把最近 60 秒带骨架叠层的视频缓存到服务器，观看端按缓存视频平滑播放；缓存超过 30 分钟会自动清理。` : "摄像头启动后会持续识别正手、反手、发球、截击、高压、切削、挑高球与未知动作。系统会用 24 帧时间窗口统一动作，再把稳定动作写入片段、训练记录与评分；分析过程中会自动录制“视频画面 + 骨架/关键点叠层”的合成回放，并按 60 秒分段归档进视频库。开启虚拟形象后，画面中的人体可切换为 10 个轻量动物替身，或 4 个免费的全身 3D Avatar 示例覆盖显示。"}

@@ -2156,20 +2707,36 @@ export default function LiveCamera() {
-
稳定动作
-
{heroAction.label}
+
+ 稳定动作 +
+
+ {heroAction.label} +
-
原始候选
-
{rawActionMeta.label}
+
+ 原始候选 +
+
+ {rawActionMeta.label} +
-
识别时长
-
{formatDuration(displayDurationMs)}
+
+ 识别时长 +
+
+ {formatDuration(displayDurationMs)} +
-
稳定窗口
-
{displayStabilityMeta.windowFrames}/{ACTION_WINDOW_FRAMES}
+
+ 稳定窗口 +
+
+ {displayStabilityMeta.windowFrames}/{ACTION_WINDOW_FRAMES} +
@@ -2187,19 +2754,24 @@ export default function LiveCamera() { muted autoPlay /> - {runtimeRole === "viewer" && viewerFrameSrc ? ( - 同步中的实时分析画面 { + playsInline + muted + autoPlay + loop + preload="auto" + onLoadedData={() => { setViewerConnected(true); setViewerError(""); }} onError={() => { setViewerConnected(false); - setViewerError("持有端正在上传同步画面，正在自动重试..."); + setViewerError("服务端缓存视频暂不可用，正在自动刷新..."); }} /> ) : null} @@ -2212,9 +2784,17 @@ export default function LiveCamera() {
-
{runtimeRole === "viewer" ? "等待同步画面" : "摄像头未启动"}
+
+ {runtimeRole === "viewer" + ? "等待同步画面" + : "摄像头未启动"} +
- {runtimeRole === "viewer" ? `${viewerModeLabel}，当前设备只能观看同步内容。` : "先完成拍摄校准，再开启自动动作识别。"} + {runtimeRole === "viewer" + ? viewerBufferReady + ? `${viewerModeLabel}，当前设备只能观看同步内容；正在载入最近 60 秒缓存回放。` + : `${viewerModeLabel}，当前设备只能观看同步内容；持有端累计满 60 秒缓存后会自动出现平滑视频。` + : "先完成拍摄校准，再开启自动动作识别。"}
{runtimeRole === "viewer" ? ( @@ -2222,8 +2802,12 @@ export default function LiveCamera() { data-testid="live-camera-viewer-button" onClick={() => { if (runtimeSession?.mediaSessionId) { - void startViewerStream(runtimeSession.mediaSessionId).catch((error: any) => { - toast.error(`同步观看连接失败: ${error?.message || "未知错误"}`); + void startViewerStream( + runtimeSession.mediaSessionId + ).catch((error: any) => { + toast.error( + `同步观看连接失败: ${error?.message || "未知错误"}` + ); }); } }} @@ -2231,10 +2815,14 @@ export default function LiveCamera() { disabled={!runtimeSession?.mediaSessionId} > - {viewerConnected ? "刷新同步" : "获取同步画面"} + {viewerConnected ? "刷新缓存" : "获取缓存视频"} ) : ( - @@ -2271,20 +2859,32 @@ export default function LiveCamera() { ) : null} - {cameraActive && zoomState.supported && runtimeRole !== "viewer" ? renderZoomOverlay() : null} + {cameraActive && zoomState.supported && runtimeRole !== "viewer" + ? renderZoomOverlay() + : null} - {(hasVideoFeed || saving) ? ( + {hasVideoFeed || saving ? (
-
稳定动作
-
{heroAction.label}
-
原始候选 {rawActionMeta.label}
+
+ 稳定动作 +
+
+ {heroAction.label} +
+
+ 原始候选 {rawActionMeta.label} +
-
稳定窗口
+
+ 稳定窗口 +
- {displayStabilityMeta.windowFrames}/{ACTION_WINDOW_FRAMES} · {Math.round(displayStabilityMeta.windowShare * 100)}% + {displayStabilityMeta.windowFrames}/ + {ACTION_WINDOW_FRAMES} ·{" "} + {Math.round(displayStabilityMeta.windowShare * 100)}%
{saving @@ -2300,10 +2900,12 @@ export default function LiveCamera() {
-
+
setAvatarKey(value as AvatarKey)} disabled={runtimeRole === "viewer"}> +
+ 形象预设 +
+
-
扩展别名
+
+ 扩展别名 +
setAvatarPrompt(event.target.value)} + onChange={event => setAvatarPrompt(event.target.value)} placeholder="例如 狐狸 / panda coach / BeachKing / Juanita" className="h-12 rounded-2xl border-border/60" disabled={runtimeRole === "viewer"} @@ -2415,12 +3086,21 @@ export default function LiveCamera() {
-
免费 3D 全身范例
+
+ 免费 3D 全身范例 +
- 这 4 个示例来自 Open Source Avatars 的 CC0 集合，当前已处理成轻量透明素材用于实时覆盖；后续若切换到 VRM/three-vrm，可继续沿用同一批模型源。 + 这 4 个示例来自 Open Source Avatars 的 CC0 + 集合，当前已处理成轻量透明素材用于实时覆盖；后续若切换到 + VRM/three-vrm，可继续沿用同一批模型源。
- CC0 · Open Source Avatars + + CC0 · Open Source Avatars +
{fullBodyAvatarPresets.map(renderAvatarShowcaseCard)} @@ -2433,18 +3113,25 @@ export default function LiveCamera() { 拍摄与流量设置 - 默认使用节省流量模式；动作切换会经过 24 帧稳定窗口确认后再入库。 + + 默认使用节省流量模式；动作切换会经过 24 帧稳定窗口确认后再入库。 +
{Object.entries(CAMERA_QUALITY_PRESETS).map(([key, preset]) => { const active = qualityPreset === key; - const disabled = analyzing || saving || runtimeRole === "viewer"; + const disabled = + analyzing || saving || runtimeRole === "viewer"; return ( ); })} @@ -2464,7 +3157,8 @@ export default function LiveCamera() {
当前采集规格
- {CAMERA_QUALITY_PRESETS[qualityPreset].subtitle} · 分析录制码率会随模式同步切换，默认优先节省流量。 + {CAMERA_QUALITY_PRESETS[qualityPreset].subtitle} · + 分析录制码率会随模式同步切换，默认优先节省流量。
@@ -2484,7 +3178,7 @@ export default function LiveCamera() { min={zoomState.min} max={zoomState.max} step={zoomState.step} - onValueChange={(value) => { + onValueChange={value => { if (typeof value[0] === "number") { void updateZoom(value[0]); } @@ -2504,7 +3198,8 @@ export default function LiveCamera() { 连续动作区间 - 只保留通过稳定窗口确认后的动作区间，单段最长 10 秒，方便后续查看和回放。 + 只保留通过稳定窗口确认后的动作区间，单段最长 10 + 秒，方便后续查看和回放。 @@ -2517,10 +3212,14 @@ export default function LiveCamera() { > 全部片段 - {actionStats.map((item) => ( + {actionStats.map(item => ( @@ -2748,8 +3565,17 @@ export default function LiveCamera() {
-