hao/tennis-training-hub
1
0
派生 0
代码 工单 合并请求 工作流 软件包 项目 发布 百科 活动

feat relay live viewer frames through media service

浏览代码
这个提交包含在：
cryptocommuniums-afk
2026-03-16 22:43:08 +08:00
父节点 bacd712dbc
当前提交 bb46d26c0e
共修改 4 个文件，包含 220 行新增和 127 行删除
下载 Patch 文件 下载 Diff 文件 展开所有文件 折叠所有文件

14
client/src/lib/media.ts
查看文件

@@ -51,6 +51,8 @@ export type MediaSession = {
streamConnected: boolean;
lastStreamAt?: string;
viewerCount?: number;
liveFrameUrl?: string;
liveFrameUpdatedAt?: string;
playback: {
webmUrl?: string;
mp4Url?: string;
@@ -131,6 +133,14 @@ export async function signalMediaViewerSession(sessionId: string, payload: { sdp
});
}
export async function uploadMediaLiveFrame(sessionId: string, blob: Blob) {
return request<{ session: MediaSession }>(`/sessions/${sessionId}/live-frame`, {
method: "POST",
headers: { "Content-Type": blob.type || "image/jpeg" },
body: blob,
});
}
export async function uploadMediaSegment(
sessionId: string,
sequence: number,
@@ -173,6 +183,10 @@ export async function getMediaSession(sessionId: string) {
return request<{ session: MediaSession }>(`/sessions/${sessionId}`);
}
export function getMediaAssetUrl(path: string) {
return `${MEDIA_BASE}${path.startsWith("/") ? path : `/${path}`}`;
}
export function formatRecordingTime(milliseconds: number) {
const totalSeconds = Math.max(0, Math.floor(milliseconds / 1000));
const minutes = Math.floor(totalSeconds / 60);

222
client/src/pages/LiveCamera.tsx
查看文件

@@ -2,8 +2,8 @@ import { useAuth } from "@/_core/hooks/useAuth";
import { trpc } from "@/lib/trpc";
import {
createMediaSession,
signalMediaSession,
signalMediaViewerSession,
getMediaAssetUrl,
uploadMediaLiveFrame,
} from "@/lib/media";
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
import { Badge } from "@/components/ui/badge";
@@ -193,23 +193,6 @@ const CAMERA_QUALITY_PRESETS: Record<CameraQualityPreset, { label: string; subti
},
};
function waitForIceGathering(peer: RTCPeerConnection) {
if (peer.iceGatheringState === "complete") {
return Promise.resolve();
}
return new Promise<void>((resolve) => {
const handleStateChange = () => {
if (peer.iceGatheringState === "complete") {
peer.removeEventListener("icegatheringstatechange", handleStateChange);
resolve();
}
};
peer.addEventListener("icegatheringstatechange", handleStateChange);
});
}
function clamp(value: number, min: number, max: number) {
return Math.max(min, Math.min(max, value));
}
@@ -567,12 +550,11 @@ export default function LiveCamera() {
const streamRef = useRef<MediaStream | null>(null);
const poseRef = useRef<any>(null);
const compositeCanvasRef = useRef<HTMLCanvasElement | null>(null);
const broadcastPeerRef = useRef<RTCPeerConnection | null>(null);
const broadcastStreamRef = useRef<MediaStream | null>(null);
const broadcastSessionIdRef = useRef<string | null>(null);
const viewerPeerRef = useRef<RTCPeerConnection | null>(null);
const viewerSessionIdRef = useRef<string | null>(null);
const viewerRetryTimerRef = useRef<number>(0);
const frameRelayTimerRef = useRef<number>(0);
const frameRelayInFlightRef = useRef(false);
const runtimeIdRef = useRef<number | null>(null);
const heartbeatTimerRef = useRef<number>(0);
const recorderRef = useRef<MediaRecorder | null>(null);
@@ -635,6 +617,7 @@ export default function LiveCamera() {
const [archivedVideoCount, setArchivedVideoCount] = useState(0);
const [viewerConnected, setViewerConnected] = useState(false);
const [viewerError, setViewerError] = useState("");
const [viewerFrameVersion, setViewerFrameVersion] = useState(0);
const resolvedAvatarKey = useMemo(
() => resolveAvatarKeyFromPrompt(avatarPrompt, avatarKey),
@@ -909,17 +892,47 @@ export default function LiveCamera() {
recentSegments: segmentsRef.current.slice(-5),
}), [facing, mobile, qualityPreset, runtimeSession?.title]);
const uploadLiveFrame = useCallback(async (sessionId: string) => {
const compositeCanvas = ensureCompositeCanvas();
if (!compositeCanvas || frameRelayInFlightRef.current) {
return;
}
renderCompositeFrame();
frameRelayInFlightRef.current = true;
try {
const blob = await new Promise<Blob | null>((resolve) => {
compositeCanvas.toBlob(resolve, "image/jpeg", mobile ? 0.7 : 0.76);
});
if (!blob) {
return;
}
await uploadMediaLiveFrame(sessionId, blob);
} finally {
frameRelayInFlightRef.current = false;
}
}, [ensureCompositeCanvas, mobile, renderCompositeFrame]);
const startFrameRelayLoop = useCallback((sessionId: string) => {
broadcastSessionIdRef.current = sessionId;
if (frameRelayTimerRef.current) {
window.clearInterval(frameRelayTimerRef.current);
frameRelayTimerRef.current = 0;
}
void uploadLiveFrame(sessionId);
frameRelayTimerRef.current = window.setInterval(() => {
void uploadLiveFrame(sessionId);
}, 900);
}, [uploadLiveFrame]);
const closeBroadcastPeer = useCallback(() => {
broadcastSessionIdRef.current = null;
if (broadcastPeerRef.current) {
broadcastPeerRef.current.onconnectionstatechange = null;
broadcastPeerRef.current.close();
broadcastPeerRef.current = null;
}
if (broadcastStreamRef.current) {
broadcastStreamRef.current.getTracks().forEach((track) => track.stop());
broadcastStreamRef.current = null;
if (frameRelayTimerRef.current) {
window.clearInterval(frameRelayTimerRef.current);
frameRelayTimerRef.current = 0;
}
frameRelayInFlightRef.current = false;
}, []);
const closeViewerPeer = useCallback(() => {
@@ -928,14 +941,11 @@ export default function LiveCamera() {
viewerRetryTimerRef.current = 0;
}
viewerSessionIdRef.current = null;
if (viewerPeerRef.current) {
viewerPeerRef.current.ontrack = null;
viewerPeerRef.current.onconnectionstatechange = null;
viewerPeerRef.current.close();
viewerPeerRef.current = null;
if (videoRef.current && !cameraActive) {
videoRef.current.srcObject = null;
}
setViewerConnected(false);
}, []);
}, [cameraActive]);
const releaseRuntime = useCallback(async (phase: RuntimeSnapshot["phase"]) => {
if (!runtimeIdRef.current) return;
@@ -989,8 +999,8 @@ export default function LiveCamera() {
}
const compositeCanvas = ensureCompositeCanvas();
if (!compositeCanvas || typeof compositeCanvas.captureStream !== "function") {
throw new Error("当前浏览器不支持同步观看推流");
if (!compositeCanvas) {
throw new Error("当前浏览器不支持同步观看画面");
}
renderCompositeFrame();
@@ -1009,84 +1019,21 @@ export default function LiveCamera() {
});
const sessionId = sessionResponse.session.id;
const stream = compositeCanvas.captureStream(mobile ? 24 : 30);
broadcastStreamRef.current = stream;
const peer = new RTCPeerConnection({
iceServers: [{ urls: ["stun:stun.l.google.com:19302"] }],
});
broadcastPeerRef.current = peer;
stream.getTracks().forEach((track) => peer.addTrack(track, stream));
const offer = await peer.createOffer();
await peer.setLocalDescription(offer);
await waitForIceGathering(peer);
const answer = await signalMediaSession(sessionId, {
sdp: peer.localDescription?.sdp || "",
type: peer.localDescription?.type || "offer",
});
await peer.setRemoteDescription({
type: answer.type as RTCSdpType,
sdp: answer.sdp,
});
startFrameRelayLoop(sessionId);
return sessionId;
}, [ensureCompositeCanvas, facing, mobile, qualityPreset, renderCompositeFrame, user?.id]);
}, [ensureCompositeCanvas, facing, mobile, qualityPreset, renderCompositeFrame, startFrameRelayLoop, user?.id]);
const startViewerStream = useCallback(async (mediaSessionId: string) => {
if (viewerSessionIdRef.current === mediaSessionId && viewerPeerRef.current) {
if (viewerSessionIdRef.current === mediaSessionId && viewerConnected) {
setViewerFrameVersion(Date.now());
return;
}
closeViewerPeer();
setViewerError("");
const peer = new RTCPeerConnection({
iceServers: [{ urls: ["stun:stun.l.google.com:19302"] }],
});
viewerPeerRef.current = peer;
viewerSessionIdRef.current = mediaSessionId;
peer.addTransceiver("video", { direction: "recvonly" });
peer.ontrack = (event) => {
const nextStream = event.streams[0] ?? new MediaStream([event.track]);
if (videoRef.current) {
videoRef.current.srcObject = nextStream;
void videoRef.current.play().catch(() => undefined);
}
setViewerConnected(true);
};
peer.onconnectionstatechange = () => {
if (peer.connectionState === "failed" || peer.connectionState === "closed" || peer.connectionState === "disconnected") {
setViewerConnected(false);
}
};
const offer = await peer.createOffer();
await peer.setLocalDescription(offer);
await waitForIceGathering(peer);
try {
const answer = await signalMediaViewerSession(mediaSessionId, {
sdp: peer.localDescription?.sdp || "",
type: peer.localDescription?.type || "offer",
});
await peer.setRemoteDescription({
type: answer.type as RTCSdpType,
sdp: answer.sdp,
});
} catch (error) {
if (viewerPeerRef.current === peer) {
closeViewerPeer();
}
throw error;
}
}, [closeViewerPeer]);
setViewerFrameVersion(Date.now());
}, [closeViewerPeer, viewerConnected]);
const stopCamera = useCallback(() => {
if (animationRef.current) {
@@ -1137,24 +1084,26 @@ export default function LiveCamera() {
}
void startViewerStream(runtimeSession.mediaSessionId).catch((error: any) => {
const message = error?.message || "同步画面连接失败";
if (/409|viewer stream not ready/i.test(message)) {
setViewerError("持有端正在准备同步画面，正在自动重试...");
if (!viewerRetryTimerRef.current) {
viewerRetryTimerRef.current = window.setTimeout(() => {
viewerRetryTimerRef.current = 0;
void runtimeQuery.refetch();
}, 1200);
}
return;
}
setViewerError(message);
setViewerError(error?.message || "同步画面连接失败");
});
if (viewerRetryTimerRef.current) {
window.clearInterval(viewerRetryTimerRef.current);
viewerRetryTimerRef.current = 0;
}
viewerRetryTimerRef.current = window.setInterval(() => {
setViewerFrameVersion(Date.now());
}, 900);
return () => {
if (viewerRetryTimerRef.current) {
window.clearInterval(viewerRetryTimerRef.current);
viewerRetryTimerRef.current = 0;
}
};
}, [
cameraActive,
closeViewerPeer,
runtimeQuery.refetch,
runtimeQuery.dataUpdatedAt,
runtimeRole,
runtimeSession?.mediaSessionId,
startViewerStream,
@@ -1726,6 +1675,9 @@ export default function LiveCamera() {
const displayRuntimeTitle = runtimeRole === "viewer"
? (runtimeSnapshot?.title ?? runtimeSession?.title ?? "其他设备实时分析")
: (runtimeSession?.title ?? `实时分析 ${ACTION_META[currentAction].label}`);
const viewerFrameSrc = runtimeRole === "viewer" && runtimeSession?.mediaSessionId
? getMediaAssetUrl(`/assets/sessions/${runtimeSession.mediaSessionId}/live-frame.jpg?ts=${viewerFrameVersion || runtimeSnapshot?.updatedAt || Date.now()}`)
: "";
const hasVideoFeed = cameraActive || viewerConnected;
const heroAction = ACTION_META[displayAction];
const rawActionMeta = ACTION_META[displayRawAction];
@@ -1736,8 +1688,8 @@ export default function LiveCamera() {
const fullBodyAvatarPresets = AVATAR_PRESETS.filter((preset) => preset.category === "full-body-3d");
const previewTitle = runtimeRole === "viewer"
? viewerConnected
? `${runtimeSyncLabel} · 同步观看中`
: "正在连接同步画面"
? `${runtimeSyncLabel} · 服务端同步中`
: "正在获取服务端同步画面"
: analyzing
? displayStabilityMeta.pending && pendingActionMeta
? `${pendingActionMeta.label} 切换确认中`
@@ -2030,7 +1982,7 @@ export default function LiveCamera() {
<Monitor className="h-4 w-4" />
<AlertTitle>同步观看模式</AlertTitle>
<AlertDescription>
{viewerModeLabel}。当前设备不会占用本地摄像头，也不能再次开启分析；视频、动作、评分与会话信息会按心跳自动同步，允许 1 秒级延迟。
{viewerModeLabel}。当前设备不会占用本地摄像头，也不能再次开启分析；同步画面会通过 media 服务中转，动作、评分与会话信息会按心跳自动同步，允许 1 秒级延迟。
</AlertDescription>
</Alert>
) : null}
@@ -2082,7 +2034,7 @@ export default function LiveCamera() {
<h1 className="text-3xl font-semibold tracking-tight">{displayRuntimeTitle}</h1>
<p className="mt-2 max-w-2xl text-sm leading-6 text-white/70">
{runtimeRole === "viewer"
? `当前正在同步 ${displayDeviceKind === "mobile" ? "移动端" : "桌面端"} ${displayFacing === "environment" ? "后置/主摄视角" : "前置视角"} 画面。视频、动作、评分、最近区间、虚拟形象和会话状态会自动跟随持有端刷新，允许少量网络延迟。`
? `当前正在通过服务端中转同步 ${displayDeviceKind === "mobile" ? "移动端" : "桌面端"} ${displayFacing === "environment" ? "后置/主摄视角" : "前置视角"} 画面。同步画面、动作、评分、最近区间、虚拟形象和会话状态会自动跟随持有端刷新，允许少量网络延迟。`
: "摄像头启动后会持续识别正手、反手、发球、截击、高压、切削、挑高球与未知动作。系统会用 24 帧时间窗口统一动作，再把稳定动作写入片段、训练记录与评分；分析过程中会自动录制“视频画面 + 骨架/关键点叠层”的合成回放，并按 60 秒分段归档进视频库。开启虚拟形象后，画面中的人体可切换为 10 个轻量动物替身，或 4 个免费的全身 3D Avatar 示例覆盖显示。"}
</p>
</div>
@@ -2116,11 +2068,27 @@ export default function LiveCamera() {
<div className="relative aspect-[16/10] overflow-hidden bg-black sm:aspect-video">
<video
ref={videoRef}
className={`absolute inset-0 h-full w-full object-contain ${immersivePreview ? "opacity-0" : ""}`}
className={`absolute inset-0 h-full w-full object-contain ${immersivePreview || runtimeRole === "viewer" ? "opacity-0" : ""}`}
playsInline
muted
autoPlay
/>
{runtimeRole === "viewer" && viewerFrameSrc ? (
<img
key={viewerFrameSrc}
src={viewerFrameSrc}
alt="同步中的实时分析画面"
className="absolute inset-0 h-full w-full object-contain"
onLoad={() => {
setViewerConnected(true);
setViewerError("");
}}
onError={() => {
setViewerConnected(false);
setViewerError("持有端正在上传同步画面，正在自动重试...");
}}
/>
) : null}
<canvas
ref={canvasRef}
className={`pointer-events-none absolute inset-0 h-full w-full object-contain ${runtimeRole === "viewer" ? "hidden" : analyzing ? "" : "opacity-70"}`}
@@ -2149,7 +2117,7 @@ export default function LiveCamera() {
disabled={!runtimeSession?.mediaSessionId}
>
<Monitor className="mr-2 h-4 w-4" />
{viewerConnected ? "重新同步" : "同步观看"}
{viewerConnected ? "刷新同步" : "获取同步画面"}
</Button>
) : (
<Button data-testid="live-camera-start-button" onClick={() => setShowSetupGuide(true)} className="rounded-2xl">