From 4fb2d092d77e1a33b0d0f53167c4d349c3ca471a Mon Sep 17 00:00:00 2001 From: cryptocommuniums-afk Date: Mon, 16 Mar 2026 11:59:51 +0800 Subject: [PATCH] Add auto archived overlay recordings for live analysis --- client/src/lib/changelog.ts | 20 +++ client/src/lib/liveCamera.ts | 32 +++-- client/src/pages/LiveCamera.tsx | 232 ++++++++++++++++++++++++++------ docs/CHANGELOG.md | 12 +- server/routers.ts | 4 +- tests/e2e/app.spec.ts | 22 +++ tests/e2e/helpers/mockApp.ts | 115 +++++++++++++++- 7 files changed, 377 insertions(+), 60 deletions(-) diff --git a/client/src/lib/changelog.ts b/client/src/lib/changelog.ts index 892ced1..724d2f1 100644 --- a/client/src/lib/changelog.ts +++ b/client/src/lib/changelog.ts @@ -8,6 +8,26 @@ export type ChangeLogEntry = { }; export const CHANGE_LOG_ENTRIES: ChangeLogEntry[] = [ + { + version: "2026.03.16-live-analysis-overlay-archive", + releaseDate: "2026-03-16", + repoVersion: "e3fe9a8 + local changes", + summary: "实时分析新增 60 秒自动归档录像，录制内容会保留骨架、关键点和虚拟形象叠层，并同步进入视频库。", + features: [ + "实时分析开始后会自动录制合成画布，每 60 秒自动切段归档", + "归档录像会保留原视频、骨架线、关键点和当前虚拟形象覆盖效果", + "归档片段会自动写入视频库，标签显示为“实时分析”", + "删除视频库中的实时分析录像时，不会删除已写入的实时分析数据和训练记录", + "线上 smoke 已确认 `https://te.hao.work/` 已切换到本次新构建，`/live-camera`、`/videos`、`/changelog` 页面均可正常访问", + ], + tests: [ + "pnpm check", + "pnpm test", + "pnpm build", + "pnpm test:e2e", + "Playwright smoke: 真实站点登录 H1，完成 /live-camera 引导、开始/结束分析，并确认 /videos 可见实时分析条目", + ], + }, { version: "2026.03.15-live-analysis-leave-hint", releaseDate: "2026-03-15", diff --git a/client/src/lib/liveCamera.ts b/client/src/lib/liveCamera.ts index 8234075..3dd7035 100644 --- a/client/src/lib/liveCamera.ts +++ b/client/src/lib/liveCamera.ts @@ -662,18 +662,22 @@ function drawFullFigureAvatar( drawLimbs(ctx, anchors, visual.limbStroke); } -export function drawLiveCameraOverlay( - canvas: HTMLCanvasElement | null, +export function renderLiveCameraOverlayToContext( + ctx: CanvasRenderingContext2D | null, + width: number, + height: number, landmarks: PosePoint[] | undefined, avatarState?: AvatarRenderState, + options?: { clear?: boolean }, ) { - const ctx = canvas?.getContext("2d"); - if (!canvas || !ctx) return; - ctx.clearRect(0, 0, canvas.width, canvas.height); + if (!ctx) return; + if (options?.clear !== false) { + ctx.clearRect(0, 0, width, height); + } if (!landmarks) return; if (avatarState?.enabled) { - const anchors = getAvatarAnchors(landmarks, canvas.width, canvas.height); + const anchors = getAvatarAnchors(landmarks, width, height); if (anchors) { const sprite = getAvatarImage(avatarState.avatarKey); const visual = AVATAR_VISUALS[avatarState.avatarKey]; @@ -715,8 +719,8 @@ export function drawLiveCameraOverlay( const end = landmarks[to]; if (!start || !end || (start.visibility ?? 1) < 0.25 || (end.visibility ?? 1) < 0.25) return; ctx.beginPath(); - ctx.moveTo(start.x * canvas.width, start.y * canvas.height); - ctx.lineTo(end.x * canvas.width, end.y * canvas.height); + ctx.moveTo(start.x * width, start.y * height); + ctx.lineTo(end.x * width, end.y * height); ctx.stroke(); }); @@ -724,7 +728,17 @@ export function drawLiveCameraOverlay( if ((point.visibility ?? 1) < 0.25) return; ctx.fillStyle = index >= 11 && index <= 16 ? "rgba(253, 224, 71, 0.95)" : "rgba(255,255,255,0.88)"; ctx.beginPath(); - ctx.arc(point.x * canvas.width, point.y * canvas.height, index >= 11 && index <= 16 ? 5 : 4, 0, Math.PI * 2); + ctx.arc(point.x * width, point.y * height, index >= 11 && index <= 16 ? 5 : 4, 0, Math.PI * 2); ctx.fill(); }); } + +export function drawLiveCameraOverlay( + canvas: HTMLCanvasElement | null, + landmarks: PosePoint[] | undefined, + avatarState?: AvatarRenderState, +) { + const ctx = canvas?.getContext("2d"); + if (!canvas || !ctx) return; + renderLiveCameraOverlayToContext(ctx, canvas.width, canvas.height, landmarks, avatarState, { clear: true }); +} diff --git a/client/src/pages/LiveCamera.tsx b/client/src/pages/LiveCamera.tsx index 2d89184..0a98754 100644 --- a/client/src/pages/LiveCamera.tsx +++ b/client/src/pages/LiveCamera.tsx @@ -20,6 +20,7 @@ import { createStableActionState, drawLiveCameraOverlay, getAvatarPreset, + renderLiveCameraOverlayToContext, resolveAvatarKeyFromPrompt, stabilizeActionStream, type AvatarKey, @@ -80,6 +81,14 @@ type ActionSegment = { clipLabel: string; }; +type ArchivedAnalysisVideo = { + videoId: number; + url: string; + sequence: number; + durationMs: number; + title: string; +}; + type Point = { x: number; y: number; @@ -122,6 +131,7 @@ const SETUP_STEPS = [ const SEGMENT_MAX_MS = 10_000; const MERGE_GAP_MS = 900; const MIN_SEGMENT_MS = 1_200; +const ANALYSIS_RECORDING_SEGMENT_MS = 60_000; const CAMERA_QUALITY_PRESETS: Record = { economy: { label: "节省流量", @@ -482,10 +492,17 @@ export default function LiveCamera() { const canvasRef = useRef(null); const streamRef = useRef(null); const poseRef = useRef(null); + const compositeCanvasRef = useRef(null); const recorderRef = useRef(null); + const recorderStreamRef = useRef(null); const recorderMimeTypeRef = useRef("video/webm"); const recorderChunksRef = useRef([]); - const recorderStopPromiseRef = useRef | null>(null); + const recorderStopPromiseRef = useRef | null>(null); + const recorderSegmentStartedAtRef = useRef(0); + const recorderSequenceRef = useRef(0); + const recorderRotateTimerRef = useRef(0); + const recorderUploadQueueRef = useRef(Promise.resolve()); + const archivedVideosRef = useRef([]); const analyzingRef = useRef(false); const animationRef = useRef(0); const sessionStartedAtRef = useRef(0); @@ -525,6 +542,7 @@ export default function LiveCamera() { const [avatarEnabled, setAvatarEnabled] = useState(false); const [avatarKey, setAvatarKey] = useState("gorilla"); const [avatarPrompt, setAvatarPrompt] = useState(""); + const [archivedVideoCount, setArchivedVideoCount] = useState(0); const resolvedAvatarKey = useMemo( () => resolveAvatarKeyFromPrompt(avatarPrompt, avatarKey), @@ -536,6 +554,7 @@ export default function LiveCamera() { onSuccess: () => { utils.profile.stats.invalidate(); utils.analysis.liveSessionList.invalidate(); + utils.video.list.invalidate(); utils.record.list.invalidate(); utils.achievement.list.invalidate(); utils.rating.current.invalidate(); @@ -621,16 +640,94 @@ export default function LiveCamera() { } }, [cameraActive, immersivePreview]); + const ensureCompositeCanvas = useCallback(() => { + if (typeof document === "undefined") { + return null; + } + if (!compositeCanvasRef.current) { + compositeCanvasRef.current = document.createElement("canvas"); + } + return compositeCanvasRef.current; + }, []); + + const renderCompositeFrame = useCallback((landmarks?: Point[]) => { + const video = videoRef.current; + const compositeCanvas = ensureCompositeCanvas(); + if (!video || !compositeCanvas || video.videoWidth <= 0 || video.videoHeight <= 0) { + return; + } + + if (compositeCanvas.width !== video.videoWidth || compositeCanvas.height !== video.videoHeight) { + compositeCanvas.width = video.videoWidth; + compositeCanvas.height = video.videoHeight; + } + + const ctx = compositeCanvas.getContext("2d"); + if (!ctx) return; + + ctx.clearRect(0, 0, compositeCanvas.width, compositeCanvas.height); + ctx.drawImage(video, 0, 0, compositeCanvas.width, compositeCanvas.height); + renderLiveCameraOverlayToContext( + ctx, + compositeCanvas.width, + compositeCanvas.height, + landmarks, + avatarRenderRef.current, + { clear: false }, + ); + }, [ensureCompositeCanvas]); + + const queueArchivedVideoUpload = useCallback(async (blob: Blob, sequence: number, durationMs: number) => { + const format = recorderMimeTypeRef.current.includes("mp4") ? "mp4" : "webm"; + const title = `实时分析录像 ${formatDateTimeShanghai(new Date(), { + year: undefined, + second: undefined, + })} · 第 ${sequence} 段`; + + recorderUploadQueueRef.current = recorderUploadQueueRef.current + .then(async () => { + const fileBase64 = await blobToBase64(blob); + const uploaded = await uploadMutation.mutateAsync({ + title, + format, + fileSize: blob.size, + duration: Math.max(1, Math.round(durationMs / 1000)), + exerciseType: "live_analysis", + fileBase64, + }); + const nextVideo: ArchivedAnalysisVideo = { + videoId: uploaded.videoId, + url: uploaded.url, + sequence, + durationMs, + title, + }; + archivedVideosRef.current = [...archivedVideosRef.current, nextVideo].sort((a, b) => a.sequence - b.sequence); + setArchivedVideoCount(archivedVideosRef.current.length); + }) + .catch((error: any) => { + toast.error(`分析录像第 ${sequence} 段归档失败: ${error?.message || "未知错误"}`); + }); + + return recorderUploadQueueRef.current; + }, [uploadMutation]); + const stopSessionRecorder = useCallback(async () => { const recorder = recorderRef.current; - if (!recorder) return null; + if (recorderRotateTimerRef.current) { + window.clearTimeout(recorderRotateTimerRef.current); + recorderRotateTimerRef.current = 0; + } + if (!recorder) { + await recorderUploadQueueRef.current; + return; + } const stopPromise = recorderStopPromiseRef.current; if (recorder.state !== "inactive") { recorder.stop(); } - recorderRef.current = null; - recorderStopPromiseRef.current = null; - return stopPromise ?? null; + await (stopPromise ?? Promise.resolve()); + await recorderUploadQueueRef.current; }, []); const stopCamera = useCallback(() => { @@ -659,6 +756,9 @@ export default function LiveCamera() { setRawAction("unknown"); setStabilityMeta(createEmptyStabilizedActionMeta()); setZoomState(readTrackZoomState(null)); + archivedVideosRef.current = []; + recorderSequenceRef.current = 0; + setArchivedVideoCount(0); setCameraActive(false); }, [stopSessionRecorder]); @@ -796,21 +896,35 @@ export default function LiveCamera() { currentSegmentRef.current = createSegment(frame.action, elapsedMs, frame); }, [flushSegment]); - const startSessionRecorder = useCallback((stream: MediaStream) => { + const startSessionRecorder = useCallback(function startSessionRecorderInternal() { if (typeof MediaRecorder === "undefined") { recorderRef.current = null; - recorderStopPromiseRef.current = Promise.resolve(null); + recorderStopPromiseRef.current = Promise.resolve(); return; } + const compositeCanvas = ensureCompositeCanvas(); + if (!compositeCanvas || typeof compositeCanvas.captureStream !== "function") { + recorderRef.current = null; + recorderStopPromiseRef.current = Promise.resolve(); + return; + } + + renderCompositeFrame(); recorderChunksRef.current = []; const mimeType = pickRecorderMimeType(); recorderMimeTypeRef.current = mimeType; - const recorder = new MediaRecorder(stream, { + if (!recorderStreamRef.current) { + recorderStreamRef.current = compositeCanvas.captureStream(mobile ? 24 : 30); + } + const recorder = new MediaRecorder(recorderStreamRef.current, { mimeType, videoBitsPerSecond: getLiveAnalysisBitrate(qualityPreset, mobile), }); recorderRef.current = recorder; + const sequence = recorderSequenceRef.current + 1; + recorderSequenceRef.current = sequence; + recorderSegmentStartedAtRef.current = Date.now(); recorder.ondataavailable = (event) => { if (event.data && event.data.size > 0) { @@ -820,14 +934,32 @@ export default function LiveCamera() { recorderStopPromiseRef.current = new Promise((resolve) => { recorder.onstop = () => { + const durationMs = Math.max(0, Date.now() - recorderSegmentStartedAtRef.current); const type = recorderMimeTypeRef.current.includes("mp4") ? "video/mp4" : "video/webm"; const blob = recorderChunksRef.current.length > 0 ? new Blob(recorderChunksRef.current, { type }) : null; - resolve(blob); + recorderChunksRef.current = []; + recorderRef.current = null; + recorderStopPromiseRef.current = null; + if (blob && blob.size > 0 && durationMs > 0) { + void queueArchivedVideoUpload(blob, sequence, durationMs); + } + if (analyzingRef.current) { + startSessionRecorderInternal(); + } else if (recorderStreamRef.current) { + recorderStreamRef.current.getTracks().forEach((track) => track.stop()); + recorderStreamRef.current = null; + } + resolve(); }; }); - recorder.start(1000); - }, [mobile, qualityPreset]); + recorder.start(); + recorderRotateTimerRef.current = window.setTimeout(() => { + if (recorder.state === "recording") { + recorder.stop(); + } + }, ANALYSIS_RECORDING_SEGMENT_MS); + }, [ensureCompositeCanvas, mobile, qualityPreset, queueArchivedVideoUpload, renderCompositeFrame]); const persistSession = useCallback(async () => { const endedAt = Date.now(); @@ -871,27 +1003,9 @@ export default function LiveCamera() { ? volatilitySamplesRef.current.reduce((sum, value) => sum + value, 0) / volatilitySamplesRef.current.length : 0; const avatarState = avatarRenderRef.current; - - let uploadedVideo: { videoId: number; url: string } | null = null; - const recordedBlob = await stopSessionRecorder(); - if (recordedBlob && recordedBlob.size > 0) { - const format = recorderMimeTypeRef.current.includes("mp4") ? "mp4" : "webm"; - const fileBase64 = await blobToBase64(recordedBlob); - uploadedVideo = await uploadMutation.mutateAsync({ - title: `实时分析 ${formatDateTimeShanghai(new Date(), { - year: undefined, - second: undefined, - })}`, - format, - fileSize: recordedBlob.size, - exerciseType: dominantAction, - fileBase64, - }); - } - - if (finalSegments.length === 0) { - return; - } + await stopSessionRecorder(); + const archivedVideos = [...archivedVideosRef.current].sort((a, b) => a.sequence - b.sequence); + const primaryArchivedVideo = archivedVideos[0] ?? null; await saveLiveSessionMutation.mutateAsync({ title: `实时分析 ${ACTION_META[dominantAction].label}`, @@ -921,6 +1035,9 @@ export default function LiveCamera() { rawActionVolatility: Number(averageRawVolatility.toFixed(4)), avatarEnabled: avatarState.enabled, avatarKey: avatarState.enabled ? avatarState.avatarKey : null, + autoRecordingEnabled: true, + autoRecordingSegmentMs: ANALYSIS_RECORDING_SEGMENT_MS, + archivedVideos, mobile, }, segments: finalSegments.map((segment) => ({ @@ -937,10 +1054,10 @@ export default function LiveCamera() { keyFrames: segment.keyFrames, clipLabel: segment.clipLabel, })), - videoId: uploadedVideo?.videoId, - videoUrl: uploadedVideo?.url, + videoId: primaryArchivedVideo?.videoId, + videoUrl: primaryArchivedVideo?.url, }); - }, [flushSegment, liveScore, mobile, saveLiveSessionMutation, sessionMode, stopSessionRecorder, uploadMutation]); + }, [flushSegment, liveScore, mobile, saveLiveSessionMutation, sessionMode, stopSessionRecorder]); const startAnalysis = useCallback(async () => { if (!cameraActive || !videoRef.current || !streamRef.current) { @@ -961,6 +1078,9 @@ export default function LiveCamera() { stableActionStateRef.current = createStableActionState(); frameSamplesRef.current = []; volatilitySamplesRef.current = []; + archivedVideosRef.current = []; + recorderSequenceRef.current = 0; + setArchivedVideoCount(0); sessionStartedAtRef.current = Date.now(); setCurrentAction("unknown"); setRawAction("unknown"); @@ -968,7 +1088,7 @@ export default function LiveCamera() { setFeedback([]); setStabilityMeta(createEmptyStabilizedActionMeta()); setDurationMs(0); - startSessionRecorder(streamRef.current); + startSessionRecorder(); try { const testFactory = ( @@ -1002,6 +1122,7 @@ export default function LiveCamera() { } drawLiveCameraOverlay(canvas, results.poseLandmarks, avatarRenderRef.current); + renderCompositeFrame(results.poseLandmarks); if (!results.poseLandmarks) return; const frameTimestamp = performance.now(); @@ -1063,7 +1184,7 @@ export default function LiveCamera() { await stopSessionRecorder(); toast.error(`实时分析启动失败: ${error?.message || "未知错误"}`); } - }, [appendFrameToSegment, cameraActive, saving, startSessionRecorder, stopSessionRecorder]); + }, [appendFrameToSegment, cameraActive, renderCompositeFrame, saving, startSessionRecorder, stopSessionRecorder]); const stopAnalysis = useCallback(async () => { if (!analyzingRef.current) return; @@ -1084,7 +1205,7 @@ export default function LiveCamera() { } await persistSession(); setLeaveStatus("safe"); - toast.success("实时分析已保存，并同步写入训练记录"); + toast.success(`实时分析已保存，并同步写入训练记录${archivedVideosRef.current.length > 0 ? `；已归档 ${archivedVideosRef.current.length} 段分析录像` : ""}`); await liveSessionsQuery.refetch(); } catch (error: any) { setLeaveStatus("failed"); @@ -1345,7 +1466,7 @@ export default function LiveCamera() { 正在保存分析结果 - 视频、动作区间和训练记录正在提交，请暂时停留当前页面；保存完成后会提示你可以离开。 + 实时分析录像、动作区间和训练记录正在提交，请暂时停留当前页面；保存完成后会提示你可以离开。 ) : null} @@ -1382,6 +1503,10 @@ export default function LiveCamera() {