From bb46d26c0eb2b07cca44a10bde8cd9f4e18f08b3 Mon Sep 17 00:00:00 2001 From: cryptocommuniums-afk Date: Mon, 16 Mar 2026 22:43:08 +0800 Subject: [PATCH] feat relay live viewer frames through media service --- client/src/lib/media.ts | 14 ++ client/src/pages/LiveCamera.tsx | 222 ++++++++++++++------------------ media/main.go | 69 ++++++++++ media/main_test.go | 42 ++++++ 4 files changed, 220 insertions(+), 127 deletions(-) diff --git a/client/src/lib/media.ts b/client/src/lib/media.ts index 0afb79f..f2d3352 100644 --- a/client/src/lib/media.ts +++ b/client/src/lib/media.ts @@ -51,6 +51,8 @@ export type MediaSession = { streamConnected: boolean; lastStreamAt?: string; viewerCount?: number; + liveFrameUrl?: string; + liveFrameUpdatedAt?: string; playback: { webmUrl?: string; mp4Url?: string; @@ -131,6 +133,14 @@ export async function signalMediaViewerSession(sessionId: string, payload: { sdp }); } +export async function uploadMediaLiveFrame(sessionId: string, blob: Blob) { + return request<{ session: MediaSession }>(`/sessions/${sessionId}/live-frame`, { + method: "POST", + headers: { "Content-Type": blob.type || "image/jpeg" }, + body: blob, + }); +} + export async function uploadMediaSegment( sessionId: string, sequence: number, @@ -173,6 +183,10 @@ export async function getMediaSession(sessionId: string) { return request<{ session: MediaSession }>(`/sessions/${sessionId}`); } +export function getMediaAssetUrl(path: string) { + return `${MEDIA_BASE}${path.startsWith("/") ? path : `/${path}`}`; +} + export function formatRecordingTime(milliseconds: number) { const totalSeconds = Math.max(0, Math.floor(milliseconds / 1000)); const minutes = Math.floor(totalSeconds / 60); diff --git a/client/src/pages/LiveCamera.tsx b/client/src/pages/LiveCamera.tsx index 6e175d8..85fbe93 100644 --- a/client/src/pages/LiveCamera.tsx +++ b/client/src/pages/LiveCamera.tsx @@ -2,8 +2,8 @@ import { useAuth } from "@/_core/hooks/useAuth"; import { trpc } from "@/lib/trpc"; import { createMediaSession, - signalMediaSession, - signalMediaViewerSession, + getMediaAssetUrl, + uploadMediaLiveFrame, } from "@/lib/media"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Badge } from "@/components/ui/badge"; @@ -193,23 +193,6 @@ const CAMERA_QUALITY_PRESETS: Record((resolve) => { - const handleStateChange = () => { - if (peer.iceGatheringState === "complete") { - peer.removeEventListener("icegatheringstatechange", handleStateChange); - resolve(); - } - }; - - peer.addEventListener("icegatheringstatechange", handleStateChange); - }); -} - function clamp(value: number, min: number, max: number) { return Math.max(min, Math.min(max, value)); } @@ -567,12 +550,11 @@ export default function LiveCamera() { const streamRef = useRef(null); const poseRef = useRef(null); const compositeCanvasRef = useRef(null); - const broadcastPeerRef = useRef(null); - const broadcastStreamRef = useRef(null); const broadcastSessionIdRef = useRef(null); - const viewerPeerRef = useRef(null); const viewerSessionIdRef = useRef(null); const viewerRetryTimerRef = useRef(0); + const frameRelayTimerRef = useRef(0); + const frameRelayInFlightRef = useRef(false); const runtimeIdRef = useRef(null); const heartbeatTimerRef = useRef(0); const recorderRef = useRef(null); @@ -635,6 +617,7 @@ export default function LiveCamera() { const [archivedVideoCount, setArchivedVideoCount] = useState(0); const [viewerConnected, setViewerConnected] = useState(false); const [viewerError, setViewerError] = useState(""); + const [viewerFrameVersion, setViewerFrameVersion] = useState(0); const resolvedAvatarKey = useMemo( () => resolveAvatarKeyFromPrompt(avatarPrompt, avatarKey), @@ -909,17 +892,47 @@ export default function LiveCamera() { recentSegments: segmentsRef.current.slice(-5), }), [facing, mobile, qualityPreset, runtimeSession?.title]); + const uploadLiveFrame = useCallback(async (sessionId: string) => { + const compositeCanvas = ensureCompositeCanvas(); + if (!compositeCanvas || frameRelayInFlightRef.current) { + return; + } + + renderCompositeFrame(); + frameRelayInFlightRef.current = true; + try { + const blob = await new Promise((resolve) => { + compositeCanvas.toBlob(resolve, "image/jpeg", mobile ? 0.7 : 0.76); + }); + if (!blob) { + return; + } + await uploadMediaLiveFrame(sessionId, blob); + } finally { + frameRelayInFlightRef.current = false; + } + }, [ensureCompositeCanvas, mobile, renderCompositeFrame]); + + const startFrameRelayLoop = useCallback((sessionId: string) => { + broadcastSessionIdRef.current = sessionId; + if (frameRelayTimerRef.current) { + window.clearInterval(frameRelayTimerRef.current); + frameRelayTimerRef.current = 0; + } + + void uploadLiveFrame(sessionId); + frameRelayTimerRef.current = window.setInterval(() => { + void uploadLiveFrame(sessionId); + }, 900); + }, [uploadLiveFrame]); + const closeBroadcastPeer = useCallback(() => { broadcastSessionIdRef.current = null; - if (broadcastPeerRef.current) { - broadcastPeerRef.current.onconnectionstatechange = null; - broadcastPeerRef.current.close(); - broadcastPeerRef.current = null; - } - if (broadcastStreamRef.current) { - broadcastStreamRef.current.getTracks().forEach((track) => track.stop()); - broadcastStreamRef.current = null; + if (frameRelayTimerRef.current) { + window.clearInterval(frameRelayTimerRef.current); + frameRelayTimerRef.current = 0; } + frameRelayInFlightRef.current = false; }, []); const closeViewerPeer = useCallback(() => { @@ -928,14 +941,11 @@ export default function LiveCamera() { viewerRetryTimerRef.current = 0; } viewerSessionIdRef.current = null; - if (viewerPeerRef.current) { - viewerPeerRef.current.ontrack = null; - viewerPeerRef.current.onconnectionstatechange = null; - viewerPeerRef.current.close(); - viewerPeerRef.current = null; + if (videoRef.current && !cameraActive) { + videoRef.current.srcObject = null; } setViewerConnected(false); - }, []); + }, [cameraActive]); const releaseRuntime = useCallback(async (phase: RuntimeSnapshot["phase"]) => { if (!runtimeIdRef.current) return; @@ -989,8 +999,8 @@ export default function LiveCamera() { } const compositeCanvas = ensureCompositeCanvas(); - if (!compositeCanvas || typeof compositeCanvas.captureStream !== "function") { - throw new Error("当前浏览器不支持同步观看推流"); + if (!compositeCanvas) { + throw new Error("当前浏览器不支持同步观看画面"); } renderCompositeFrame(); @@ -1009,84 +1019,21 @@ export default function LiveCamera() { }); const sessionId = sessionResponse.session.id; - const stream = compositeCanvas.captureStream(mobile ? 24 : 30); - broadcastStreamRef.current = stream; - - const peer = new RTCPeerConnection({ - iceServers: [{ urls: ["stun:stun.l.google.com:19302"] }], - }); - broadcastPeerRef.current = peer; - - stream.getTracks().forEach((track) => peer.addTrack(track, stream)); - - const offer = await peer.createOffer(); - await peer.setLocalDescription(offer); - await waitForIceGathering(peer); - - const answer = await signalMediaSession(sessionId, { - sdp: peer.localDescription?.sdp || "", - type: peer.localDescription?.type || "offer", - }); - - await peer.setRemoteDescription({ - type: answer.type as RTCSdpType, - sdp: answer.sdp, - }); - + startFrameRelayLoop(sessionId); return sessionId; - }, [ensureCompositeCanvas, facing, mobile, qualityPreset, renderCompositeFrame, user?.id]); + }, [ensureCompositeCanvas, facing, mobile, qualityPreset, renderCompositeFrame, startFrameRelayLoop, user?.id]); const startViewerStream = useCallback(async (mediaSessionId: string) => { - if (viewerSessionIdRef.current === mediaSessionId && viewerPeerRef.current) { + if (viewerSessionIdRef.current === mediaSessionId && viewerConnected) { + setViewerFrameVersion(Date.now()); return; } closeViewerPeer(); setViewerError(""); - - const peer = new RTCPeerConnection({ - iceServers: [{ urls: ["stun:stun.l.google.com:19302"] }], - }); - viewerPeerRef.current = peer; viewerSessionIdRef.current = mediaSessionId; - peer.addTransceiver("video", { direction: "recvonly" }); - - peer.ontrack = (event) => { - const nextStream = event.streams[0] ?? new MediaStream([event.track]); - if (videoRef.current) { - videoRef.current.srcObject = nextStream; - void videoRef.current.play().catch(() => undefined); - } - setViewerConnected(true); - }; - - peer.onconnectionstatechange = () => { - if (peer.connectionState === "failed" || peer.connectionState === "closed" || peer.connectionState === "disconnected") { - setViewerConnected(false); - } - }; - - const offer = await peer.createOffer(); - await peer.setLocalDescription(offer); - await waitForIceGathering(peer); - - try { - const answer = await signalMediaViewerSession(mediaSessionId, { - sdp: peer.localDescription?.sdp || "", - type: peer.localDescription?.type || "offer", - }); - - await peer.setRemoteDescription({ - type: answer.type as RTCSdpType, - sdp: answer.sdp, - }); - } catch (error) { - if (viewerPeerRef.current === peer) { - closeViewerPeer(); - } - throw error; - } - }, [closeViewerPeer]); + setViewerFrameVersion(Date.now()); + }, [closeViewerPeer, viewerConnected]); const stopCamera = useCallback(() => { if (animationRef.current) { @@ -1137,24 +1084,26 @@ export default function LiveCamera() { } void startViewerStream(runtimeSession.mediaSessionId).catch((error: any) => { - const message = error?.message || "同步画面连接失败"; - if (/409|viewer stream not ready/i.test(message)) { - setViewerError("持有端正在准备同步画面，正在自动重试..."); - if (!viewerRetryTimerRef.current) { - viewerRetryTimerRef.current = window.setTimeout(() => { - viewerRetryTimerRef.current = 0; - void runtimeQuery.refetch(); - }, 1200); - } - return; - } - setViewerError(message); + setViewerError(error?.message || "同步画面连接失败"); }); + + if (viewerRetryTimerRef.current) { + window.clearInterval(viewerRetryTimerRef.current); + viewerRetryTimerRef.current = 0; + } + viewerRetryTimerRef.current = window.setInterval(() => { + setViewerFrameVersion(Date.now()); + }, 900); + + return () => { + if (viewerRetryTimerRef.current) { + window.clearInterval(viewerRetryTimerRef.current); + viewerRetryTimerRef.current = 0; + } + }; }, [ cameraActive, closeViewerPeer, - runtimeQuery.refetch, - runtimeQuery.dataUpdatedAt, runtimeRole, runtimeSession?.mediaSessionId, startViewerStream, @@ -1726,6 +1675,9 @@ export default function LiveCamera() { const displayRuntimeTitle = runtimeRole === "viewer" ? (runtimeSnapshot?.title ?? runtimeSession?.title ?? "其他设备实时分析") : (runtimeSession?.title ?? `实时分析 ${ACTION_META[currentAction].label}`); + const viewerFrameSrc = runtimeRole === "viewer" && runtimeSession?.mediaSessionId + ? getMediaAssetUrl(`/assets/sessions/${runtimeSession.mediaSessionId}/live-frame.jpg?ts=${viewerFrameVersion || runtimeSnapshot?.updatedAt || Date.now()}`) + : ""; const hasVideoFeed = cameraActive || viewerConnected; const heroAction = ACTION_META[displayAction]; const rawActionMeta = ACTION_META[displayRawAction]; @@ -1736,8 +1688,8 @@ export default function LiveCamera() { const fullBodyAvatarPresets = AVATAR_PRESETS.filter((preset) => preset.category === "full-body-3d"); const previewTitle = runtimeRole === "viewer" ? viewerConnected - ? `${runtimeSyncLabel} · 同步观看中` - : "正在连接同步画面" + ? `${runtimeSyncLabel} · 服务端同步中` + : "正在获取服务端同步画面" : analyzing ? displayStabilityMeta.pending && pendingActionMeta ? `${pendingActionMeta.label} 切换确认中` @@ -2030,7 +1982,7 @@ export default function LiveCamera() { 同步观看模式 - {viewerModeLabel}。当前设备不会占用本地摄像头，也不能再次开启分析；视频、动作、评分与会话信息会按心跳自动同步，允许 1 秒级延迟。 + {viewerModeLabel}。当前设备不会占用本地摄像头，也不能再次开启分析；同步画面会通过 media 服务中转，动作、评分与会话信息会按心跳自动同步，允许 1 秒级延迟。 ) : null} @@ -2082,7 +2034,7 @@ export default function LiveCamera() {

{displayRuntimeTitle}

{runtimeRole === "viewer" - ? `当前正在同步 ${displayDeviceKind === "mobile" ? "移动端" : "桌面端"} ${displayFacing === "environment" ? "后置/主摄视角" : "前置视角"} 画面。视频、动作、评分、最近区间、虚拟形象和会话状态会自动跟随持有端刷新，允许少量网络延迟。` + ? `当前正在通过服务端中转同步 ${displayDeviceKind === "mobile" ? "移动端" : "桌面端"} ${displayFacing === "environment" ? "后置/主摄视角" : "前置视角"} 画面。同步画面、动作、评分、最近区间、虚拟形象和会话状态会自动跟随持有端刷新，允许少量网络延迟。` : "摄像头启动后会持续识别正手、反手、发球、截击、高压、切削、挑高球与未知动作。系统会用 24 帧时间窗口统一动作，再把稳定动作写入片段、训练记录与评分；分析过程中会自动录制“视频画面 + 骨架/关键点叠层”的合成回放，并按 60 秒分段归档进视频库。开启虚拟形象后，画面中的人体可切换为 10 个轻量动物替身，或 4 个免费的全身 3D Avatar 示例覆盖显示。"}

@@ -2116,11 +2068,27 @@ export default function LiveCamera() {