比较提交
7 次代码提交
09b1b95e2c
...
main
| 作者 | SHA1 | 提交日期 | |
|---|---|---|---|
|
06b9701e03 | ||
|
|
8e9e4915e2 | ||
|
|
634a4704c7 | ||
|
|
bb46d26c0e | ||
|
|
bacd712dbc | ||
|
|
78a7c755e3 | ||
|
|
a211562860 |
@@ -9,6 +9,13 @@ export type CameraZoomState = {
|
||||
focusMode: string;
|
||||
};
|
||||
|
||||
export type CameraRequestResult = {
|
||||
stream: MediaStream;
|
||||
appliedFacingMode: "user" | "environment";
|
||||
audioEnabled: boolean;
|
||||
usedFallback: boolean;
|
||||
};
|
||||
|
||||
type NumericRange = {
|
||||
min: number;
|
||||
max: number;
|
||||
@@ -66,6 +73,98 @@ export function getCameraVideoConstraints(
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeVideoConstraintCandidate(candidate: MediaTrackConstraints | true) {
|
||||
if (candidate === true) {
|
||||
return { label: "camera-any", video: true as const };
|
||||
}
|
||||
|
||||
return {
|
||||
label: JSON.stringify(candidate),
|
||||
video: candidate,
|
||||
};
|
||||
}
|
||||
|
||||
function createFallbackVideoCandidates(
|
||||
facingMode: "user" | "environment",
|
||||
isMobile: boolean,
|
||||
preset: CameraQualityPreset,
|
||||
) {
|
||||
const base = getCameraVideoConstraints(facingMode, isMobile, preset);
|
||||
const alternateFacing = facingMode === "environment" ? "user" : "environment";
|
||||
const lowRes = {
|
||||
facingMode,
|
||||
width: { ideal: isMobile ? 640 : 960 },
|
||||
height: { ideal: isMobile ? 360 : 540 },
|
||||
} satisfies MediaTrackConstraints;
|
||||
const lowResAlternate = {
|
||||
facingMode: alternateFacing,
|
||||
width: { ideal: isMobile ? 640 : 960 },
|
||||
height: { ideal: isMobile ? 360 : 540 },
|
||||
} satisfies MediaTrackConstraints;
|
||||
const anyCamera = {
|
||||
width: { ideal: isMobile ? 640 : 960 },
|
||||
height: { ideal: isMobile ? 360 : 540 },
|
||||
} satisfies MediaTrackConstraints;
|
||||
|
||||
const candidates = [
|
||||
normalizeVideoConstraintCandidate(base),
|
||||
normalizeVideoConstraintCandidate({
|
||||
...base,
|
||||
frameRate: undefined,
|
||||
}),
|
||||
normalizeVideoConstraintCandidate(lowRes),
|
||||
normalizeVideoConstraintCandidate(lowResAlternate),
|
||||
normalizeVideoConstraintCandidate(anyCamera),
|
||||
normalizeVideoConstraintCandidate(true),
|
||||
];
|
||||
|
||||
const deduped = new Map<string, { video: MediaTrackConstraints | true }>();
|
||||
candidates.forEach((candidate) => {
|
||||
if (!deduped.has(candidate.label)) {
|
||||
deduped.set(candidate.label, { video: candidate.video });
|
||||
}
|
||||
});
|
||||
return Array.from(deduped.values());
|
||||
}
|
||||
|
||||
export async function requestCameraStream(options: {
|
||||
facingMode: "user" | "environment";
|
||||
isMobile: boolean;
|
||||
preset: CameraQualityPreset;
|
||||
audio?: false | MediaTrackConstraints;
|
||||
}) {
|
||||
const videoCandidates = createFallbackVideoCandidates(options.facingMode, options.isMobile, options.preset);
|
||||
const audioCandidates = options.audio ? [options.audio, false] : [false];
|
||||
let lastError: unknown = null;
|
||||
|
||||
for (const audio of audioCandidates) {
|
||||
for (let index = 0; index < videoCandidates.length; index += 1) {
|
||||
const video = videoCandidates[index]?.video ?? true;
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ video, audio });
|
||||
const videoTrack = stream.getVideoTracks()[0] || null;
|
||||
const settings = (
|
||||
videoTrack && typeof (videoTrack as MediaStreamTrack & { getSettings?: () => unknown }).getSettings === "function"
|
||||
? (videoTrack as MediaStreamTrack & { getSettings: () => unknown }).getSettings()
|
||||
: {}
|
||||
) as Record<string, unknown>;
|
||||
const appliedFacingMode = settings.facingMode === "user" ? "user" : settings.facingMode === "environment" ? "environment" : options.facingMode;
|
||||
|
||||
return {
|
||||
stream,
|
||||
appliedFacingMode,
|
||||
audioEnabled: stream.getAudioTracks().length > 0,
|
||||
usedFallback: index > 0 || audio === false && Boolean(options.audio),
|
||||
} satisfies CameraRequestResult;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError instanceof Error ? lastError : new Error("无法访问摄像头");
|
||||
}
|
||||
|
||||
export function getLiveAnalysisBitrate(preset: CameraQualityPreset, isMobile: boolean) {
|
||||
switch (preset) {
|
||||
case "economy":
|
||||
|
||||
@@ -8,6 +8,56 @@ export type ChangeLogEntry = {
|
||||
};
|
||||
|
||||
export const CHANGE_LOG_ENTRIES: ChangeLogEntry[] = [
|
||||
{
|
||||
version: "2026.03.16-live-camera-runtime-refresh",
|
||||
releaseDate: "2026-03-16",
|
||||
repoVersion: "8e9e491",
|
||||
summary: "修复实时分析页偶发残留在同步观看状态、标题乱码，以及摄像头预览绑定波动导致的启动失败。",
|
||||
features: [
|
||||
"live-camera 在打开拍摄引导、启用摄像头、开始分析前，都会先向服务端强制刷新 runtime 状态，避免旧的 viewer 锁残留导致本机明明已释放却仍无法启动",
|
||||
"同步观看标题新增乱码恢复逻辑，可自动把 UTF-8 被误按 Latin-1 显示的标题恢复成正常中文，避免出现 `æœ...` 一类异常标题",
|
||||
"摄像头启动链路改为以 `getUserMedia` 成功为准；即使本地预览 `<video>` 的 `srcObject` 或 `play()` 在当前浏览器里短暂失败，也不会直接把整次启动判死",
|
||||
"e2e mock 的媒体流补齐为带假视频轨道的流对象，并把 viewer 回归改为校验“服务端 relay、无 viewer-signal”行为，减少和旧 P2P 逻辑混淆",
|
||||
],
|
||||
tests: [
|
||||
"pnpm exec playwright test tests/e2e/app.spec.ts --grep \"live camera page exposes camera startup controls|live camera switches into viewer mode when another device already owns analysis|live camera recovers mojibake viewer titles before rendering|live camera no longer opens viewer peer retries when server relay is active\"",
|
||||
"pnpm build",
|
||||
"部署后线上 smoke: `https://te.hao.work/live-camera` 登录 H1 后可见空闲态“启动摄像头”入口，确认不再被残留 viewer 锁卡住；公开站点前端资源为 `assets/index-33wVjC4p.js` 与 `assets/index-tNGuStgv.css`",
|
||||
],
|
||||
},
|
||||
{
|
||||
version: "2026.03.16-live-viewer-server-relay",
|
||||
releaseDate: "2026-03-16",
|
||||
repoVersion: "bb46d26",
|
||||
summary: "实时分析同步观看改为由 media 服务中转帧图，不再依赖浏览器之间的 P2P 视频连接。",
|
||||
features: [
|
||||
"owner 端现在会把带骨架、关键点和虚拟形象叠层的合成画布压缩成 JPEG 并持续上传到 media 服务",
|
||||
"viewer 端改为直接拉取 media 服务中的最新同步帧图，不再建立 WebRTC viewer peer 连接，因此跨网络和多端观看更稳定",
|
||||
"同步观看模式文案改为明确提示“通过 media 服务中转”，等待同步时也会自动轮询最新画面",
|
||||
"media 服务新增 live-frame 上传与静态分发能力，并记录最近同步帧的更新时间，方便后续扩展成更高频的服务端中转流",
|
||||
],
|
||||
tests: [
|
||||
"cd media && go test ./...",
|
||||
"pnpm build",
|
||||
"playwright-skill 线上 smoke: 先用 media 服务创建 relay session、上传 live-frame，并把 H1 的 `live_analysis_runtime` 注入为 active viewer 场景；随后访问 `https://te.hao.work/live-camera`，确认页面进入“同步观看模式”、同步帧来自 `/media/assets/sessions/.../live-frame.jpg`，且 `viewer-signal` 请求数为 0",
|
||||
],
|
||||
},
|
||||
{
|
||||
version: "2026.03.16-camera-startup-fallbacks",
|
||||
releaseDate: "2026-03-16",
|
||||
repoVersion: "a211562",
|
||||
summary: "修复部分设备上摄像头因后置镜头约束、分辨率约束或麦克风不可用而直接启动失败的问题。",
|
||||
features: [
|
||||
"live-camera 与 recorder 改为共用分级降级的摄像头请求流程，会在当前画质失败时自动降分辨率、降约束并回退到兼容镜头",
|
||||
"当设备不支持默认后置摄像头或当前镜头不可用时，页面会自动切换到实际可用的镜头方向，避免直接报错后卡死在未启动状态",
|
||||
"recorder 预览启动不再被麦克风权限或麦克风设备异常整体拖死；麦克风不可用时会自动回退到仅视频模式",
|
||||
"兼容模式命中时前端会给出明确提示，方便区分“已自动降级成功”与“仍然无法访问摄像头”的场景",
|
||||
],
|
||||
tests: [
|
||||
"pnpm build",
|
||||
"部署后线上 smoke: `https://te.hao.work/` 已提供 `assets/index-CRxtWK07.js` 与 `assets/index-tNGuStgv.css`；通过注入 `getUserMedia` 回归验证 `/live-camera` 首轮高约束失败后会自动切到兼容摄像头模式，`/recorder` 在麦克风不可用时会自动回退到仅视频模式并继续启动预览",
|
||||
],
|
||||
},
|
||||
{
|
||||
version: "2026.03.16-live-analysis-viewer-full-sync",
|
||||
releaseDate: "2026-03-16",
|
||||
|
||||
@@ -51,6 +51,8 @@ export type MediaSession = {
|
||||
streamConnected: boolean;
|
||||
lastStreamAt?: string;
|
||||
viewerCount?: number;
|
||||
liveFrameUrl?: string;
|
||||
liveFrameUpdatedAt?: string;
|
||||
playback: {
|
||||
webmUrl?: string;
|
||||
mp4Url?: string;
|
||||
@@ -131,6 +133,14 @@ export async function signalMediaViewerSession(sessionId: string, payload: { sdp
|
||||
});
|
||||
}
|
||||
|
||||
export async function uploadMediaLiveFrame(sessionId: string, blob: Blob) {
|
||||
return request<{ session: MediaSession }>(`/sessions/${sessionId}/live-frame`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": blob.type || "image/jpeg" },
|
||||
body: blob,
|
||||
});
|
||||
}
|
||||
|
||||
export async function uploadMediaSegment(
|
||||
sessionId: string,
|
||||
sequence: number,
|
||||
@@ -173,6 +183,10 @@ export async function getMediaSession(sessionId: string) {
|
||||
return request<{ session: MediaSession }>(`/sessions/${sessionId}`);
|
||||
}
|
||||
|
||||
export function getMediaAssetUrl(path: string) {
|
||||
return `${MEDIA_BASE}${path.startsWith("/") ? path : `/${path}`}`;
|
||||
}
|
||||
|
||||
export function formatRecordingTime(milliseconds: number) {
|
||||
const totalSeconds = Math.max(0, Math.floor(milliseconds / 1000));
|
||||
const minutes = Math.floor(totalSeconds / 60);
|
||||
|
||||
@@ -2,8 +2,8 @@ import { useAuth } from "@/_core/hooks/useAuth";
|
||||
import { trpc } from "@/lib/trpc";
|
||||
import {
|
||||
createMediaSession,
|
||||
signalMediaSession,
|
||||
signalMediaViewerSession,
|
||||
getMediaAssetUrl,
|
||||
uploadMediaLiveFrame,
|
||||
} from "@/lib/media";
|
||||
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
@@ -17,7 +17,7 @@ import { Slider } from "@/components/ui/slider";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import { formatDateTimeShanghai } from "@/lib/time";
|
||||
import { toast } from "sonner";
|
||||
import { applyTrackZoom, type CameraQualityPreset, getCameraVideoConstraints, getLiveAnalysisBitrate, readTrackZoomState } from "@/lib/camera";
|
||||
import { applyTrackZoom, type CameraQualityPreset, getLiveAnalysisBitrate, readTrackZoomState, requestCameraStream } from "@/lib/camera";
|
||||
import {
|
||||
ACTION_WINDOW_FRAMES,
|
||||
AVATAR_PRESETS,
|
||||
@@ -193,23 +193,6 @@ const CAMERA_QUALITY_PRESETS: Record<CameraQualityPreset, { label: string; subti
|
||||
},
|
||||
};
|
||||
|
||||
function waitForIceGathering(peer: RTCPeerConnection) {
|
||||
if (peer.iceGatheringState === "complete") {
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
return new Promise<void>((resolve) => {
|
||||
const handleStateChange = () => {
|
||||
if (peer.iceGatheringState === "complete") {
|
||||
peer.removeEventListener("icegatheringstatechange", handleStateChange);
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
|
||||
peer.addEventListener("icegatheringstatechange", handleStateChange);
|
||||
});
|
||||
}
|
||||
|
||||
function clamp(value: number, min: number, max: number) {
|
||||
return Math.max(min, Math.min(max, value));
|
||||
}
|
||||
@@ -236,6 +219,35 @@ function formatDuration(ms: number) {
|
||||
return `${minutes.toString().padStart(2, "0")}:${seconds.toString().padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
function normalizeRuntimeTitle(value: string | null | undefined) {
|
||||
if (typeof value !== "string") return "";
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed) return "";
|
||||
|
||||
const suspicious = /[ÃÂÆÐÑØæåçéèêëïîôöûüœŠŽƒ€¦]/;
|
||||
if (!suspicious.test(trimmed)) {
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
try {
|
||||
const bytes = Uint8Array.from(Array.from(trimmed).map((char) => char.charCodeAt(0) & 0xff));
|
||||
const decoded = new TextDecoder("utf-8").decode(bytes).trim();
|
||||
if (!decoded || decoded === trimmed) {
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
const score = (text: string) => {
|
||||
const cjkCount = text.match(/[\u3400-\u9fff]/g)?.length ?? 0;
|
||||
const badCount = text.match(/[ÃÂÆÐÑØæåçéèêëïîôöûüœŠŽƒ€¦]/g)?.length ?? 0;
|
||||
return (cjkCount * 2) - badCount;
|
||||
};
|
||||
|
||||
return score(decoded) > score(trimmed) ? decoded : trimmed;
|
||||
} catch {
|
||||
return trimmed;
|
||||
}
|
||||
}
|
||||
|
||||
function isMobileDevice() {
|
||||
if (typeof window === "undefined") return false;
|
||||
return /Android|iPhone|iPad|iPod/i.test(navigator.userAgent) || window.matchMedia("(max-width: 768px)").matches;
|
||||
@@ -567,12 +579,11 @@ export default function LiveCamera() {
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const poseRef = useRef<any>(null);
|
||||
const compositeCanvasRef = useRef<HTMLCanvasElement | null>(null);
|
||||
const broadcastPeerRef = useRef<RTCPeerConnection | null>(null);
|
||||
const broadcastStreamRef = useRef<MediaStream | null>(null);
|
||||
const broadcastSessionIdRef = useRef<string | null>(null);
|
||||
const viewerPeerRef = useRef<RTCPeerConnection | null>(null);
|
||||
const viewerSessionIdRef = useRef<string | null>(null);
|
||||
const viewerRetryTimerRef = useRef<number>(0);
|
||||
const frameRelayTimerRef = useRef<number>(0);
|
||||
const frameRelayInFlightRef = useRef(false);
|
||||
const runtimeIdRef = useRef<number | null>(null);
|
||||
const heartbeatTimerRef = useRef<number>(0);
|
||||
const recorderRef = useRef<MediaRecorder | null>(null);
|
||||
@@ -635,6 +646,7 @@ export default function LiveCamera() {
|
||||
const [archivedVideoCount, setArchivedVideoCount] = useState(0);
|
||||
const [viewerConnected, setViewerConnected] = useState(false);
|
||||
const [viewerError, setViewerError] = useState("");
|
||||
const [viewerFrameVersion, setViewerFrameVersion] = useState(0);
|
||||
|
||||
const resolvedAvatarKey = useMemo(
|
||||
() => resolveAvatarKeyFromPrompt(avatarPrompt, avatarKey),
|
||||
@@ -665,6 +677,8 @@ export default function LiveCamera() {
|
||||
const runtimeRole = (runtimeQuery.data?.role ?? "idle") as RuntimeRole;
|
||||
const runtimeSession = (runtimeQuery.data?.runtimeSession ?? null) as RuntimeSession | null;
|
||||
const runtimeSnapshot = runtimeSession?.snapshot ?? null;
|
||||
const normalizedRuntimeTitle = normalizeRuntimeTitle(runtimeSession?.title);
|
||||
const normalizedSnapshotTitle = normalizeRuntimeTitle(runtimeSnapshot?.title);
|
||||
|
||||
useEffect(() => {
|
||||
avatarRenderRef.current = {
|
||||
@@ -780,6 +794,14 @@ export default function LiveCamera() {
|
||||
[displayVisibleSegments.length, knownRatio, liveScore?.overall, runtimeRole, runtimeSnapshot?.liveScore?.overall],
|
||||
);
|
||||
|
||||
const refreshRuntimeState = useCallback(async () => {
|
||||
const result = await runtimeQuery.refetch();
|
||||
return {
|
||||
role: (result.data?.role ?? runtimeRole) as RuntimeRole,
|
||||
runtimeSession: (result.data?.runtimeSession ?? runtimeSession) as RuntimeSession | null,
|
||||
};
|
||||
}, [runtimeQuery, runtimeRole, runtimeSession]);
|
||||
|
||||
useEffect(() => {
|
||||
navigator.mediaDevices?.enumerateDevices().then((devices) => {
|
||||
const cameras = devices.filter((device) => device.kind === "videoinput");
|
||||
@@ -889,7 +911,7 @@ export default function LiveCamera() {
|
||||
phase: phase ?? leaveStatusRef.current,
|
||||
startedAt: sessionStartedAtRef.current || undefined,
|
||||
durationMs: durationMsRef.current,
|
||||
title: runtimeSession?.title ?? `实时分析 ${ACTION_META[currentActionRef.current].label}`,
|
||||
title: normalizedRuntimeTitle || `实时分析 ${ACTION_META[currentActionRef.current].label}`,
|
||||
sessionMode: sessionModeRef.current,
|
||||
qualityPreset,
|
||||
facingMode: facing,
|
||||
@@ -907,19 +929,59 @@ export default function LiveCamera() {
|
||||
unknownSegments: segmentsRef.current.filter((segment) => segment.isUnknown).length,
|
||||
archivedVideoCount: archivedVideosRef.current.length,
|
||||
recentSegments: segmentsRef.current.slice(-5),
|
||||
}), [facing, mobile, qualityPreset, runtimeSession?.title]);
|
||||
}), [facing, mobile, normalizedRuntimeTitle, qualityPreset]);
|
||||
|
||||
const openSetupGuide = useCallback(async () => {
|
||||
const latest = await refreshRuntimeState();
|
||||
if (latest.role === "viewer") {
|
||||
setShowSetupGuide(false);
|
||||
toast.error("当前账号已有其他设备正在实时分析，请先切换到同步观看模式");
|
||||
return;
|
||||
}
|
||||
setShowSetupGuide(true);
|
||||
}, [refreshRuntimeState]);
|
||||
|
||||
const uploadLiveFrame = useCallback(async (sessionId: string) => {
|
||||
const compositeCanvas = ensureCompositeCanvas();
|
||||
if (!compositeCanvas || frameRelayInFlightRef.current) {
|
||||
return;
|
||||
}
|
||||
|
||||
renderCompositeFrame();
|
||||
frameRelayInFlightRef.current = true;
|
||||
try {
|
||||
const blob = await new Promise<Blob | null>((resolve) => {
|
||||
compositeCanvas.toBlob(resolve, "image/jpeg", mobile ? 0.7 : 0.76);
|
||||
});
|
||||
if (!blob) {
|
||||
return;
|
||||
}
|
||||
await uploadMediaLiveFrame(sessionId, blob);
|
||||
} finally {
|
||||
frameRelayInFlightRef.current = false;
|
||||
}
|
||||
}, [ensureCompositeCanvas, mobile, renderCompositeFrame]);
|
||||
|
||||
const startFrameRelayLoop = useCallback((sessionId: string) => {
|
||||
broadcastSessionIdRef.current = sessionId;
|
||||
if (frameRelayTimerRef.current) {
|
||||
window.clearInterval(frameRelayTimerRef.current);
|
||||
frameRelayTimerRef.current = 0;
|
||||
}
|
||||
|
||||
void uploadLiveFrame(sessionId);
|
||||
frameRelayTimerRef.current = window.setInterval(() => {
|
||||
void uploadLiveFrame(sessionId);
|
||||
}, 900);
|
||||
}, [uploadLiveFrame]);
|
||||
|
||||
const closeBroadcastPeer = useCallback(() => {
|
||||
broadcastSessionIdRef.current = null;
|
||||
if (broadcastPeerRef.current) {
|
||||
broadcastPeerRef.current.onconnectionstatechange = null;
|
||||
broadcastPeerRef.current.close();
|
||||
broadcastPeerRef.current = null;
|
||||
}
|
||||
if (broadcastStreamRef.current) {
|
||||
broadcastStreamRef.current.getTracks().forEach((track) => track.stop());
|
||||
broadcastStreamRef.current = null;
|
||||
if (frameRelayTimerRef.current) {
|
||||
window.clearInterval(frameRelayTimerRef.current);
|
||||
frameRelayTimerRef.current = 0;
|
||||
}
|
||||
frameRelayInFlightRef.current = false;
|
||||
}, []);
|
||||
|
||||
const closeViewerPeer = useCallback(() => {
|
||||
@@ -928,14 +990,11 @@ export default function LiveCamera() {
|
||||
viewerRetryTimerRef.current = 0;
|
||||
}
|
||||
viewerSessionIdRef.current = null;
|
||||
if (viewerPeerRef.current) {
|
||||
viewerPeerRef.current.ontrack = null;
|
||||
viewerPeerRef.current.onconnectionstatechange = null;
|
||||
viewerPeerRef.current.close();
|
||||
viewerPeerRef.current = null;
|
||||
if (videoRef.current && !cameraActive) {
|
||||
videoRef.current.srcObject = null;
|
||||
}
|
||||
setViewerConnected(false);
|
||||
}, []);
|
||||
}, [cameraActive]);
|
||||
|
||||
const releaseRuntime = useCallback(async (phase: RuntimeSnapshot["phase"]) => {
|
||||
if (!runtimeIdRef.current) return;
|
||||
@@ -989,8 +1048,8 @@ export default function LiveCamera() {
|
||||
}
|
||||
|
||||
const compositeCanvas = ensureCompositeCanvas();
|
||||
if (!compositeCanvas || typeof compositeCanvas.captureStream !== "function") {
|
||||
throw new Error("当前浏览器不支持同步观看推流");
|
||||
if (!compositeCanvas) {
|
||||
throw new Error("当前浏览器不支持同步观看画面");
|
||||
}
|
||||
|
||||
renderCompositeFrame();
|
||||
@@ -1009,84 +1068,21 @@ export default function LiveCamera() {
|
||||
});
|
||||
|
||||
const sessionId = sessionResponse.session.id;
|
||||
const stream = compositeCanvas.captureStream(mobile ? 24 : 30);
|
||||
broadcastStreamRef.current = stream;
|
||||
|
||||
const peer = new RTCPeerConnection({
|
||||
iceServers: [{ urls: ["stun:stun.l.google.com:19302"] }],
|
||||
});
|
||||
broadcastPeerRef.current = peer;
|
||||
|
||||
stream.getTracks().forEach((track) => peer.addTrack(track, stream));
|
||||
|
||||
const offer = await peer.createOffer();
|
||||
await peer.setLocalDescription(offer);
|
||||
await waitForIceGathering(peer);
|
||||
|
||||
const answer = await signalMediaSession(sessionId, {
|
||||
sdp: peer.localDescription?.sdp || "",
|
||||
type: peer.localDescription?.type || "offer",
|
||||
});
|
||||
|
||||
await peer.setRemoteDescription({
|
||||
type: answer.type as RTCSdpType,
|
||||
sdp: answer.sdp,
|
||||
});
|
||||
|
||||
startFrameRelayLoop(sessionId);
|
||||
return sessionId;
|
||||
}, [ensureCompositeCanvas, facing, mobile, qualityPreset, renderCompositeFrame, user?.id]);
|
||||
}, [ensureCompositeCanvas, facing, mobile, qualityPreset, renderCompositeFrame, startFrameRelayLoop, user?.id]);
|
||||
|
||||
const startViewerStream = useCallback(async (mediaSessionId: string) => {
|
||||
if (viewerSessionIdRef.current === mediaSessionId && viewerPeerRef.current) {
|
||||
if (viewerSessionIdRef.current === mediaSessionId && viewerConnected) {
|
||||
setViewerFrameVersion(Date.now());
|
||||
return;
|
||||
}
|
||||
|
||||
closeViewerPeer();
|
||||
setViewerError("");
|
||||
|
||||
const peer = new RTCPeerConnection({
|
||||
iceServers: [{ urls: ["stun:stun.l.google.com:19302"] }],
|
||||
});
|
||||
viewerPeerRef.current = peer;
|
||||
viewerSessionIdRef.current = mediaSessionId;
|
||||
peer.addTransceiver("video", { direction: "recvonly" });
|
||||
|
||||
peer.ontrack = (event) => {
|
||||
const nextStream = event.streams[0] ?? new MediaStream([event.track]);
|
||||
if (videoRef.current) {
|
||||
videoRef.current.srcObject = nextStream;
|
||||
void videoRef.current.play().catch(() => undefined);
|
||||
}
|
||||
setViewerConnected(true);
|
||||
};
|
||||
|
||||
peer.onconnectionstatechange = () => {
|
||||
if (peer.connectionState === "failed" || peer.connectionState === "closed" || peer.connectionState === "disconnected") {
|
||||
setViewerConnected(false);
|
||||
}
|
||||
};
|
||||
|
||||
const offer = await peer.createOffer();
|
||||
await peer.setLocalDescription(offer);
|
||||
await waitForIceGathering(peer);
|
||||
|
||||
try {
|
||||
const answer = await signalMediaViewerSession(mediaSessionId, {
|
||||
sdp: peer.localDescription?.sdp || "",
|
||||
type: peer.localDescription?.type || "offer",
|
||||
});
|
||||
|
||||
await peer.setRemoteDescription({
|
||||
type: answer.type as RTCSdpType,
|
||||
sdp: answer.sdp,
|
||||
});
|
||||
} catch (error) {
|
||||
if (viewerPeerRef.current === peer) {
|
||||
closeViewerPeer();
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}, [closeViewerPeer]);
|
||||
setViewerFrameVersion(Date.now());
|
||||
}, [closeViewerPeer, viewerConnected]);
|
||||
|
||||
const stopCamera = useCallback(() => {
|
||||
if (animationRef.current) {
|
||||
@@ -1137,24 +1133,26 @@ export default function LiveCamera() {
|
||||
}
|
||||
|
||||
void startViewerStream(runtimeSession.mediaSessionId).catch((error: any) => {
|
||||
const message = error?.message || "同步画面连接失败";
|
||||
if (/409|viewer stream not ready/i.test(message)) {
|
||||
setViewerError("持有端正在准备同步画面，正在自动重试...");
|
||||
if (!viewerRetryTimerRef.current) {
|
||||
viewerRetryTimerRef.current = window.setTimeout(() => {
|
||||
viewerRetryTimerRef.current = 0;
|
||||
void runtimeQuery.refetch();
|
||||
}, 1200);
|
||||
}
|
||||
return;
|
||||
}
|
||||
setViewerError(message);
|
||||
setViewerError(error?.message || "同步画面连接失败");
|
||||
});
|
||||
|
||||
if (viewerRetryTimerRef.current) {
|
||||
window.clearInterval(viewerRetryTimerRef.current);
|
||||
viewerRetryTimerRef.current = 0;
|
||||
}
|
||||
viewerRetryTimerRef.current = window.setInterval(() => {
|
||||
setViewerFrameVersion(Date.now());
|
||||
}, 900);
|
||||
|
||||
return () => {
|
||||
if (viewerRetryTimerRef.current) {
|
||||
window.clearInterval(viewerRetryTimerRef.current);
|
||||
viewerRetryTimerRef.current = 0;
|
||||
}
|
||||
};
|
||||
}, [
|
||||
cameraActive,
|
||||
closeViewerPeer,
|
||||
runtimeQuery.refetch,
|
||||
runtimeQuery.dataUpdatedAt,
|
||||
runtimeRole,
|
||||
runtimeSession?.mediaSessionId,
|
||||
startViewerStream,
|
||||
@@ -1213,7 +1211,8 @@ export default function LiveCamera() {
|
||||
preferredZoom = zoomTargetRef.current,
|
||||
preset: CameraQualityPreset = qualityPreset,
|
||||
) => {
|
||||
if (runtimeRole === "viewer") {
|
||||
const latest = await refreshRuntimeState();
|
||||
if (latest.role === "viewer") {
|
||||
toast.error("当前账号已有其他设备正在实时分析，请切换到同步观看模式");
|
||||
return;
|
||||
}
|
||||
@@ -1221,25 +1220,33 @@ export default function LiveCamera() {
|
||||
if (streamRef.current) {
|
||||
streamRef.current.getTracks().forEach((track) => track.stop());
|
||||
}
|
||||
|
||||
const constraints: MediaStreamConstraints = {
|
||||
video: getCameraVideoConstraints(nextFacing, mobile, preset),
|
||||
audio: false,
|
||||
};
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia(constraints);
|
||||
const { stream, appliedFacingMode, usedFallback } = await requestCameraStream({
|
||||
facingMode: nextFacing,
|
||||
isMobile: mobile,
|
||||
preset,
|
||||
});
|
||||
streamRef.current = stream;
|
||||
if (appliedFacingMode !== nextFacing) {
|
||||
setFacing(appliedFacingMode);
|
||||
}
|
||||
setCameraActive(true);
|
||||
if (videoRef.current) {
|
||||
videoRef.current.srcObject = stream;
|
||||
await videoRef.current.play();
|
||||
try {
|
||||
videoRef.current.srcObject = stream;
|
||||
await videoRef.current.play().catch(() => undefined);
|
||||
} catch {
|
||||
// Keep the camera session alive even if preview binding is flaky on the current browser.
|
||||
}
|
||||
}
|
||||
await syncZoomState(preferredZoom, stream.getVideoTracks()[0] || null);
|
||||
setCameraActive(true);
|
||||
if (usedFallback) {
|
||||
toast.info("当前设备已自动切换到兼容摄像头模式");
|
||||
}
|
||||
toast.success("摄像头已启动");
|
||||
} catch (error: any) {
|
||||
toast.error(`摄像头启动失败: ${error?.message || "未知错误"}`);
|
||||
}
|
||||
}, [facing, mobile, qualityPreset, runtimeRole, syncZoomState]);
|
||||
}, [facing, mobile, qualityPreset, refreshRuntimeState, syncZoomState]);
|
||||
|
||||
const switchCamera = useCallback(async () => {
|
||||
const nextFacing: CameraFacing = facing === "user" ? "environment" : "user";
|
||||
@@ -1464,12 +1471,13 @@ export default function LiveCamera() {
|
||||
}, [flushSegment, liveScore, mobile, saveLiveSessionMutation, sessionMode, stopSessionRecorder]);
|
||||
|
||||
const startAnalysis = useCallback(async () => {
|
||||
const latest = await refreshRuntimeState();
|
||||
if (!cameraActive || !videoRef.current || !streamRef.current) {
|
||||
toast.error("请先启动摄像头");
|
||||
return;
|
||||
}
|
||||
if (analyzingRef.current || saving) return;
|
||||
if (runtimeRole === "viewer") {
|
||||
if (latest.role === "viewer") {
|
||||
toast.error("当前设备处于同步观看模式，不能重复开启分析");
|
||||
return;
|
||||
}
|
||||
@@ -1617,10 +1625,10 @@ export default function LiveCamera() {
|
||||
appendFrameToSegment,
|
||||
cameraActive,
|
||||
closeBroadcastPeer,
|
||||
refreshRuntimeState,
|
||||
releaseRuntime,
|
||||
runtimeAcquireMutation,
|
||||
runtimeQuery,
|
||||
runtimeRole,
|
||||
saving,
|
||||
sessionMode,
|
||||
startBroadcastSession,
|
||||
@@ -1679,9 +1687,15 @@ export default function LiveCamera() {
|
||||
}, [analyzing, saving]);
|
||||
|
||||
const handleSetupComplete = useCallback(async () => {
|
||||
const latest = await refreshRuntimeState();
|
||||
if (latest.role === "viewer") {
|
||||
setShowSetupGuide(false);
|
||||
toast.error("当前账号已有其他设备正在实时分析，请切换到同步观看模式");
|
||||
return;
|
||||
}
|
||||
setShowSetupGuide(false);
|
||||
await startCamera(facing, zoomTargetRef.current, qualityPreset);
|
||||
}, [facing, qualityPreset, startCamera]);
|
||||
}, [facing, qualityPreset, refreshRuntimeState, startCamera]);
|
||||
|
||||
const displayLeaveStatus = runtimeRole === "viewer" ? (runtimeSnapshot?.phase ?? "idle") : leaveStatus;
|
||||
const displayAction = runtimeRole === "viewer" ? (runtimeSnapshot?.currentAction ?? "unknown") : currentAction;
|
||||
@@ -1720,8 +1734,11 @@ export default function LiveCamera() {
|
||||
const runtimeSyncDelayMs = runtimeRole === "viewer" ? getRuntimeSyncDelayMs(runtimeSession?.lastHeartbeatAt) : null;
|
||||
const runtimeSyncLabel = runtimeRole === "viewer" ? formatRuntimeSyncDelay(runtimeSyncDelayMs) : "";
|
||||
const displayRuntimeTitle = runtimeRole === "viewer"
|
||||
? (runtimeSnapshot?.title ?? runtimeSession?.title ?? "其他设备实时分析")
|
||||
: (runtimeSession?.title ?? `实时分析 ${ACTION_META[currentAction].label}`);
|
||||
? (normalizedSnapshotTitle || normalizedRuntimeTitle || "其他设备实时分析")
|
||||
: (normalizedRuntimeTitle || `实时分析 ${ACTION_META[currentAction].label}`);
|
||||
const viewerFrameSrc = runtimeRole === "viewer" && runtimeSession?.mediaSessionId
|
||||
? getMediaAssetUrl(`/assets/sessions/${runtimeSession.mediaSessionId}/live-frame.jpg?ts=${viewerFrameVersion || runtimeSnapshot?.updatedAt || Date.now()}`)
|
||||
: "";
|
||||
const hasVideoFeed = cameraActive || viewerConnected;
|
||||
const heroAction = ACTION_META[displayAction];
|
||||
const rawActionMeta = ACTION_META[displayRawAction];
|
||||
@@ -1732,8 +1749,8 @@ export default function LiveCamera() {
|
||||
const fullBodyAvatarPresets = AVATAR_PRESETS.filter((preset) => preset.category === "full-body-3d");
|
||||
const previewTitle = runtimeRole === "viewer"
|
||||
? viewerConnected
|
||||
? `${runtimeSyncLabel} · 同步观看中`
|
||||
: "正在连接同步画面"
|
||||
? `${runtimeSyncLabel} · 服务端同步中`
|
||||
: "正在获取服务端同步画面"
|
||||
: analyzing
|
||||
? displayStabilityMeta.pending && pendingActionMeta
|
||||
? `${pendingActionMeta.label} 切换确认中`
|
||||
@@ -1742,7 +1759,7 @@ export default function LiveCamera() {
|
||||
? "准备开始实时分析"
|
||||
: "摄像头待启动";
|
||||
|
||||
const viewerModeLabel = runtimeSession?.title || "其他设备正在实时分析";
|
||||
const viewerModeLabel = normalizedRuntimeTitle || "其他设备正在实时分析";
|
||||
|
||||
const renderPrimaryActions = (rail = false) => {
|
||||
const buttonClass = rail
|
||||
@@ -1782,7 +1799,7 @@ export default function LiveCamera() {
|
||||
<Button
|
||||
data-testid={rail ? undefined : "live-camera-toolbar-start-button"}
|
||||
className={buttonClass}
|
||||
onClick={() => setShowSetupGuide(true)}
|
||||
onClick={() => void openSetupGuide()}
|
||||
>
|
||||
<Camera className={rail ? "h-5 w-5" : "mr-2 h-4 w-4"} />
|
||||
{!rail && "启动摄像头"}
|
||||
@@ -2026,7 +2043,7 @@ export default function LiveCamera() {
|
||||
<Monitor className="h-4 w-4" />
|
||||
<AlertTitle>同步观看模式</AlertTitle>
|
||||
<AlertDescription>
|
||||
{viewerModeLabel}。当前设备不会占用本地摄像头，也不能再次开启分析；视频、动作、评分与会话信息会按心跳自动同步，允许 1 秒级延迟。
|
||||
{viewerModeLabel}。当前设备不会占用本地摄像头，也不能再次开启分析；同步画面会通过 media 服务中转，动作、评分与会话信息会按心跳自动同步，允许 1 秒级延迟。
|
||||
</AlertDescription>
|
||||
</Alert>
|
||||
) : null}
|
||||
@@ -2078,7 +2095,7 @@ export default function LiveCamera() {
|
||||
<h1 className="text-3xl font-semibold tracking-tight">{displayRuntimeTitle}</h1>
|
||||
<p className="mt-2 max-w-2xl text-sm leading-6 text-white/70">
|
||||
{runtimeRole === "viewer"
|
||||
? `当前正在同步 ${displayDeviceKind === "mobile" ? "移动端" : "桌面端"} ${displayFacing === "environment" ? "后置/主摄视角" : "前置视角"} 画面。视频、动作、评分、最近区间、虚拟形象和会话状态会自动跟随持有端刷新，允许少量网络延迟。`
|
||||
? `当前正在通过服务端中转同步 ${displayDeviceKind === "mobile" ? "移动端" : "桌面端"} ${displayFacing === "environment" ? "后置/主摄视角" : "前置视角"} 画面。同步画面、动作、评分、最近区间、虚拟形象和会话状态会自动跟随持有端刷新，允许少量网络延迟。`
|
||||
: "摄像头启动后会持续识别正手、反手、发球、截击、高压、切削、挑高球与未知动作。系统会用 24 帧时间窗口统一动作，再把稳定动作写入片段、训练记录与评分；分析过程中会自动录制“视频画面 + 骨架/关键点叠层”的合成回放，并按 60 秒分段归档进视频库。开启虚拟形象后，画面中的人体可切换为 10 个轻量动物替身，或 4 个免费的全身 3D Avatar 示例覆盖显示。"}
|
||||
</p>
|
||||
</div>
|
||||
@@ -2112,11 +2129,27 @@ export default function LiveCamera() {
|
||||
<div className="relative aspect-[16/10] overflow-hidden bg-black sm:aspect-video">
|
||||
<video
|
||||
ref={videoRef}
|
||||
className={`absolute inset-0 h-full w-full object-contain ${immersivePreview ? "opacity-0" : ""}`}
|
||||
className={`absolute inset-0 h-full w-full object-contain ${immersivePreview || runtimeRole === "viewer" ? "opacity-0" : ""}`}
|
||||
playsInline
|
||||
muted
|
||||
autoPlay
|
||||
/>
|
||||
{runtimeRole === "viewer" && viewerFrameSrc ? (
|
||||
<img
|
||||
key={viewerFrameSrc}
|
||||
src={viewerFrameSrc}
|
||||
alt="同步中的实时分析画面"
|
||||
className="absolute inset-0 h-full w-full object-contain"
|
||||
onLoad={() => {
|
||||
setViewerConnected(true);
|
||||
setViewerError("");
|
||||
}}
|
||||
onError={() => {
|
||||
setViewerConnected(false);
|
||||
setViewerError("持有端正在上传同步画面，正在自动重试...");
|
||||
}}
|
||||
/>
|
||||
) : null}
|
||||
<canvas
|
||||
ref={canvasRef}
|
||||
className={`pointer-events-none absolute inset-0 h-full w-full object-contain ${runtimeRole === "viewer" ? "hidden" : analyzing ? "" : "opacity-70"}`}
|
||||
@@ -2145,10 +2178,10 @@ export default function LiveCamera() {
|
||||
disabled={!runtimeSession?.mediaSessionId}
|
||||
>
|
||||
<Monitor className="mr-2 h-4 w-4" />
|
||||
{viewerConnected ? "重新同步" : "同步观看"}
|
||||
{viewerConnected ? "刷新同步" : "获取同步画面"}
|
||||
</Button>
|
||||
) : (
|
||||
<Button data-testid="live-camera-start-button" onClick={() => setShowSetupGuide(true)} className="rounded-2xl">
|
||||
<Button data-testid="live-camera-start-button" onClick={() => void openSetupGuide()} className="rounded-2xl">
|
||||
<Camera className="mr-2 h-4 w-4" />
|
||||
启动摄像头
|
||||
</Button>
|
||||
|
||||
@@ -31,7 +31,7 @@ import {
|
||||
recognizeActionFrame,
|
||||
stabilizeActionFrame,
|
||||
} from "@/lib/actionRecognition";
|
||||
import { applyTrackZoom, getCameraVideoConstraints, readTrackZoomState } from "@/lib/camera";
|
||||
import { applyTrackZoom, readTrackZoomState, requestCameraStream } from "@/lib/camera";
|
||||
import { formatDateTimeShanghai } from "@/lib/time";
|
||||
import {
|
||||
Activity,
|
||||
@@ -420,8 +420,10 @@ export default function Recorder() {
|
||||
streamRef.current = null;
|
||||
}
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
video: getCameraVideoConstraints(nextFacingMode, mobile, preset),
|
||||
const { stream, appliedFacingMode, audioEnabled, usedFallback } = await requestCameraStream({
|
||||
facingMode: nextFacingMode,
|
||||
isMobile: mobile,
|
||||
preset,
|
||||
audio: {
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
@@ -438,6 +440,9 @@ export default function Recorder() {
|
||||
|
||||
suppressTrackEndedRef.current = false;
|
||||
streamRef.current = stream;
|
||||
if (appliedFacingMode !== nextFacingMode) {
|
||||
setFacingMode(appliedFacingMode);
|
||||
}
|
||||
if (liveVideoRef.current) {
|
||||
liveVideoRef.current.srcObject = stream;
|
||||
await liveVideoRef.current.play();
|
||||
@@ -445,6 +450,12 @@ export default function Recorder() {
|
||||
await syncZoomState(preferredZoom, stream.getVideoTracks()[0] || null);
|
||||
setCameraError("");
|
||||
setCameraActive(true);
|
||||
if (usedFallback) {
|
||||
toast.info("当前设备已自动切换到兼容摄像头模式");
|
||||
}
|
||||
if (!audioEnabled) {
|
||||
toast.warning("麦克风不可用，已切换为仅视频模式");
|
||||
}
|
||||
return stream;
|
||||
} catch (error: any) {
|
||||
const message = error?.message || "无法访问摄像头";
|
||||
|
||||
@@ -1,5 +1,80 @@
|
||||
# Tennis Training Hub - 变更日志
|
||||
|
||||
## 2026.03.16-live-camera-runtime-refresh (2026-03-16)
|
||||
|
||||
### 功能更新
|
||||
|
||||
- `/live-camera` 在打开拍摄引导、启用摄像头、开始分析前，都会先向服务端强制刷新 runtime 状态，避免旧的同步观看锁残留导致本机明明已释放却仍无法启动
|
||||
- 新增 runtime 标题乱码恢复逻辑，可自动把 UTF-8 被误按 Latin-1 显示的标题恢复成正常中文，避免出现 `æœ...` 一类异常标题
|
||||
- 摄像头启动链路改为以 `getUserMedia` 成功为准；即使本地预览 `<video>` 的 `srcObject` 或 `play()` 在当前浏览器中短暂失败，也不会直接把整次启动判死
|
||||
- e2e mock 的媒体流补齐为带假视频轨道的流对象，并把 viewer 回归改为校验“服务端 relay、无 viewer-signal”行为，避免继续按旧 P2P 逻辑断言
|
||||
|
||||
### 测试
|
||||
|
||||
- `pnpm exec playwright test tests/e2e/app.spec.ts --grep "live camera page exposes camera startup controls|live camera switches into viewer mode when another device already owns analysis|live camera recovers mojibake viewer titles before rendering|live camera no longer opens viewer peer retries when server relay is active"`
|
||||
- `pnpm build`
|
||||
- 部署后线上 smoke：登录 `H1` 后访问 `https://te.hao.work/live-camera`，确认空闲态“启动摄像头”入口可见，不再被残留 viewer 锁卡住
|
||||
|
||||
### 线上 smoke
|
||||
|
||||
- `https://te.hao.work/` 已切换到本次新构建
|
||||
- 当前公开站点前端资源 revision：`assets/index-33wVjC4p.js` 与 `assets/index-tNGuStgv.css`
|
||||
- 真实验证已通过：登录 `H1` 后访问 `https://te.hao.work/live-camera`，页面会正常显示“摄像头未启动 / 启动摄像头”，说明旧的 viewer 锁残留不会再把空闲设备卡在同步观看模式
|
||||
|
||||
### 仓库版本
|
||||
|
||||
- `8e9e491`
|
||||
|
||||
## 2026.03.16-live-viewer-server-relay (2026-03-16)
|
||||
|
||||
### 功能更新
|
||||
|
||||
- `/live-camera` 的同步观看改为由 media 服务中转最新合成帧图，不再依赖浏览器之间的 P2P WebRTC viewer 连接
|
||||
- owner 端会把“原视频 + 骨架/关键点 + 虚拟形象”的合成画布压缩成 JPEG 并持续上传到 media 服务
|
||||
- viewer 端改为自动轮询 media 服务中的最新同步帧图，因此即使浏览器之间无法直连，也能继续看到同步画面和状态
|
||||
- 同步观看模式文案已调整为明确提示“通过 media 服务中转”，等待阶段会继续自动刷新，而不是停留在 P2P 连接失败状态
|
||||
- media 服务新增 live-frame 上传与静态分发能力，并记录最近同步帧时间，方便后续继续扩展更高频的服务端 relay
|
||||
|
||||
### 测试
|
||||
|
||||
- `cd media && go test ./...`
|
||||
- `pnpm build`
|
||||
- `playwright-skill` 线上 smoke：先用 media 服务创建 relay session、上传 live-frame，并把 `H1` 的 `live_analysis_runtime` 注入为 active viewer 场景；随后访问 `https://te.hao.work/live-camera`，确认页面进入“同步观看模式”、同步帧来自 `/media/assets/sessions/.../live-frame.jpg`，且 `viewer-signal` 请求数为 `0`
|
||||
|
||||
### 线上 smoke
|
||||
|
||||
- `https://te.hao.work/` 已切换到本次新构建
|
||||
- 当前公开站点前端资源 revision：`assets/index-BC-IupO8.js` 与 `assets/index-tNGuStgv.css`
|
||||
- 真实验证已通过：viewer 端进入“同步观看模式”后，画面由 media 服务静态分发的 `live-frame.jpg` 提供，已确认不再触发 `/viewer-signal` P2P 观看请求
|
||||
|
||||
### 仓库版本
|
||||
|
||||
- `bb46d26`
|
||||
|
||||
## 2026.03.16-camera-startup-fallbacks (2026-03-16)
|
||||
|
||||
### 功能更新
|
||||
|
||||
- 修复部分设备在 `/live-camera` 和 `/recorder` 中因默认后置镜头、分辨率或帧率约束不兼容而直接启动摄像头失败的问题
|
||||
- 摄像头请求现在会自动按当前画质、去掉高约束、低分辨率、备用镜头、任意可用镜头依次降级重试
|
||||
- `/recorder` 在麦克风不可用或麦克风权限未给出时，会自动回退到仅视频模式，不再让整次预览启动失败
|
||||
- 如果实际启用的是兼容镜头或降级模式，页面会显示提示，帮助区分“自动修复成功”与“仍然无法访问摄像头”
|
||||
|
||||
### 测试
|
||||
|
||||
- `pnpm build`
|
||||
- `playwright-skill` 线上 smoke：通过注入 `getUserMedia` 回归验证 `/live-camera` 首轮高约束失败后会自动降级到兼容摄像头模式，`/recorder` 在麦克风不可用时会自动回退到仅视频模式并继续启动预览
|
||||
|
||||
### 线上 smoke
|
||||
|
||||
- `https://te.hao.work/` 已切换到本次新构建
|
||||
- 当前公开站点前端资源 revision：`assets/index-CRxtWK07.js` 与 `assets/index-tNGuStgv.css`
|
||||
- 真实回归已通过：模拟高约束失败时，`/live-camera` 会提示“当前设备已自动切换到兼容摄像头模式”并继续启动；模拟麦克风不可用时，`/recorder` 会提示“麦克风不可用，已切换为仅视频模式”并继续显示录制入口
|
||||
|
||||
### 仓库版本
|
||||
|
||||
- `a211562`
|
||||
|
||||
## 2026.03.16-live-analysis-viewer-full-sync (2026-03-16)
|
||||
|
||||
### 功能更新
|
||||
|
||||
@@ -105,6 +105,8 @@ type Session struct {
|
||||
StreamConnected bool `json:"streamConnected"`
|
||||
LastStreamAt string `json:"lastStreamAt,omitempty"`
|
||||
ViewerCount int `json:"viewerCount"`
|
||||
LiveFrameURL string `json:"liveFrameUrl,omitempty"`
|
||||
LiveFrameUpdated string `json:"liveFrameUpdatedAt,omitempty"`
|
||||
Playback PlaybackInfo `json:"playback"`
|
||||
Segments []SegmentMeta `json:"segments"`
|
||||
Markers []Marker `json:"markers"`
|
||||
@@ -229,6 +231,14 @@ func (s *sessionStore) publicDir(id string) string {
|
||||
return filepath.Join(s.public, "sessions", id)
|
||||
}
|
||||
|
||||
func (s *sessionStore) liveFramePath(id string) string {
|
||||
return filepath.Join(s.publicDir(id), "live-frame.jpg")
|
||||
}
|
||||
|
||||
func (s *sessionStore) liveFrameURL(id string) string {
|
||||
return fmt.Sprintf("/media/assets/sessions/%s/live-frame.jpg", id)
|
||||
}
|
||||
|
||||
func (s *sessionStore) saveSession(session *Session) error {
|
||||
session.UpdatedAt = time.Now().UTC().Format(time.RFC3339)
|
||||
dir := s.sessionDir(session.ID)
|
||||
@@ -504,6 +514,12 @@ func (m *mediaServer) handleSession(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
m.handleSegmentUpload(sessionID, w, r)
|
||||
case "live-frame":
|
||||
if r.Method != http.MethodPost {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
m.handleLiveFrameUpload(sessionID, w, r)
|
||||
case "markers":
|
||||
if r.Method != http.MethodPost {
|
||||
http.NotFound(w, r)
|
||||
@@ -726,6 +742,59 @@ func (m *mediaServer) handleViewerSignal(sessionID string, w http.ResponseWriter
|
||||
})
|
||||
}
|
||||
|
||||
func (m *mediaServer) handleLiveFrameUpload(sessionID string, w http.ResponseWriter, r *http.Request) {
|
||||
if _, err := m.store.getSession(sessionID); err != nil {
|
||||
writeError(w, http.StatusNotFound, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
body := http.MaxBytesReader(w, r.Body, 4<<20)
|
||||
defer body.Close()
|
||||
|
||||
frame, err := io.ReadAll(body)
|
||||
if err != nil || len(frame) == 0 {
|
||||
writeError(w, http.StatusBadRequest, "invalid live frame payload")
|
||||
return
|
||||
}
|
||||
|
||||
publicDir := m.store.publicDir(sessionID)
|
||||
if err := os.MkdirAll(publicDir, 0o755); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "failed to create live frame directory")
|
||||
return
|
||||
}
|
||||
|
||||
tmpFile := filepath.Join(publicDir, fmt.Sprintf("live-frame-%s.tmp", randomID()))
|
||||
if err := os.WriteFile(tmpFile, frame, 0o644); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "failed to write live frame")
|
||||
return
|
||||
}
|
||||
defer os.Remove(tmpFile)
|
||||
|
||||
finalFile := m.store.liveFramePath(sessionID)
|
||||
if err := os.Rename(tmpFile, finalFile); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "failed to publish live frame")
|
||||
return
|
||||
}
|
||||
|
||||
session, err := m.store.updateSession(sessionID, func(session *Session) error {
|
||||
session.LiveFrameURL = m.store.liveFrameURL(sessionID)
|
||||
session.LiveFrameUpdated = time.Now().UTC().Format(time.RFC3339)
|
||||
session.StreamConnected = true
|
||||
session.LastStreamAt = session.LiveFrameUpdated
|
||||
if session.Status == StatusCreated || session.Status == StatusReconnecting {
|
||||
session.Status = StatusStreaming
|
||||
}
|
||||
session.LastError = ""
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "failed to update live frame session state")
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"session": session})
|
||||
}
|
||||
|
||||
func (m *mediaServer) handleSegmentUpload(sessionID string, w http.ResponseWriter, r *http.Request) {
|
||||
sequence, err := strconv.Atoi(r.URL.Query().Get("sequence"))
|
||||
if err != nil || sequence < 0 {
|
||||
|
||||
@@ -278,3 +278,45 @@ func TestViewerSignalReturnsConflictBeforePublisherTrackReady(t *testing.T) {
|
||||
t.Fatalf("expected viewer-signal 409 before video track is ready, got %d", res.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLiveFrameUploadPublishesRelayFrame(t *testing.T) {
|
||||
store, err := newSessionStore(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("newSessionStore: %v", err)
|
||||
}
|
||||
|
||||
server := newMediaServer(store)
|
||||
session, err := store.createSession(CreateSessionRequest{UserID: "1", Title: "Relay Session"})
|
||||
if err != nil {
|
||||
t.Fatalf("createSession: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/media/sessions/"+session.ID+"/live-frame", strings.NewReader("jpeg-frame"))
|
||||
req.Header.Set("Content-Type", "image/jpeg")
|
||||
res := httptest.NewRecorder()
|
||||
server.routes().ServeHTTP(res, req)
|
||||
|
||||
if res.Code != http.StatusAccepted {
|
||||
t.Fatalf("expected live-frame upload 202, got %d", res.Code)
|
||||
}
|
||||
|
||||
current, err := store.getSession(session.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("getSession: %v", err)
|
||||
}
|
||||
if current.LiveFrameURL == "" || current.LiveFrameUpdated == "" {
|
||||
t.Fatalf("expected live frame metadata to be recorded, got %#v", current)
|
||||
}
|
||||
if !current.StreamConnected {
|
||||
t.Fatalf("expected session stream connected after frame upload")
|
||||
}
|
||||
|
||||
framePath := store.liveFramePath(session.ID)
|
||||
body, err := os.ReadFile(framePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read live frame: %v", err)
|
||||
}
|
||||
if string(body) != "jpeg-frame" {
|
||||
t.Fatalf("unexpected live frame content: %q", string(body))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,7 +82,23 @@ test("live camera switches into viewer mode when another device already owns ana
|
||||
await expect(page.getByTestId("live-camera-score-overall")).toBeVisible();
|
||||
});
|
||||
|
||||
test("live camera retries viewer stream when owner track is not ready on first attempt", async ({ page }) => {
|
||||
test("live camera recovers mojibake viewer titles before rendering", async ({ page }) => {
|
||||
const state = await installAppMocks(page, { authenticated: true, liveViewerMode: true });
|
||||
const mojibakeTitle = Buffer.from("服务端同步烟雾测试", "utf8").toString("latin1");
|
||||
if (state.liveRuntime.runtimeSession) {
|
||||
state.liveRuntime.runtimeSession.title = mojibakeTitle;
|
||||
state.liveRuntime.runtimeSession.snapshot = {
|
||||
...state.liveRuntime.runtimeSession.snapshot,
|
||||
title: mojibakeTitle,
|
||||
};
|
||||
}
|
||||
|
||||
await page.goto("/live-camera");
|
||||
await expect(page.getByRole("heading", { name: "服务端同步烟雾测试" })).toBeVisible();
|
||||
await expect(page.getByText(mojibakeTitle)).toHaveCount(0);
|
||||
});
|
||||
|
||||
test("live camera no longer opens viewer peer retries when server relay is active", async ({ page }) => {
|
||||
const state = await installAppMocks(page, {
|
||||
authenticated: true,
|
||||
liveViewerMode: true,
|
||||
@@ -91,9 +107,9 @@ test("live camera retries viewer stream when owner track is not ready on first a
|
||||
|
||||
await page.goto("/live-camera");
|
||||
await expect(page.getByText("同步观看模式")).toBeVisible();
|
||||
await expect.poll(() => state.viewerSignalConflictRemaining).toBe(0);
|
||||
await expect.poll(() => state.mediaSession?.viewerCount ?? 0).toBe(1);
|
||||
await expect(page.getByText(/同步观看中|重新同步/).first()).toBeVisible();
|
||||
await expect.poll(() => state.viewerSignalConflictRemaining).toBe(1);
|
||||
await expect.poll(() => state.mediaSession?.viewerCount ?? 0).toBe(0);
|
||||
await expect(page.locator('img[alt="同步中的实时分析画面"]')).toBeVisible();
|
||||
});
|
||||
|
||||
test("live camera archives overlay videos into the library after analysis stops", async ({ page }) => {
|
||||
|
||||
@@ -866,6 +866,73 @@ export async function installAppMocks(
|
||||
return points;
|
||||
};
|
||||
|
||||
class FakeVideoTrack {
|
||||
kind = "video";
|
||||
enabled = true;
|
||||
muted = false;
|
||||
readyState = "live";
|
||||
id = "fake-video-track";
|
||||
label = "Fake Camera";
|
||||
|
||||
stop() {}
|
||||
|
||||
getSettings() {
|
||||
return {
|
||||
facingMode: "environment",
|
||||
width: 1280,
|
||||
height: 720,
|
||||
frameRate: 30,
|
||||
};
|
||||
}
|
||||
|
||||
getCapabilities() {
|
||||
return {};
|
||||
}
|
||||
|
||||
async applyConstraints() {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
class FakeAudioTrack {
|
||||
kind = "audio";
|
||||
enabled = true;
|
||||
muted = false;
|
||||
readyState = "live";
|
||||
id = "fake-audio-track";
|
||||
label = "Fake Mic";
|
||||
|
||||
stop() {}
|
||||
|
||||
getSettings() {
|
||||
return {};
|
||||
}
|
||||
|
||||
getCapabilities() {
|
||||
return {};
|
||||
}
|
||||
|
||||
async applyConstraints() {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
const createFakeMediaStream = (withAudio = false) => {
|
||||
const videoTrack = new FakeVideoTrack();
|
||||
const audioTrack = withAudio ? new FakeAudioTrack() : null;
|
||||
const tracks = audioTrack ? [videoTrack, audioTrack] : [videoTrack];
|
||||
return {
|
||||
active: true,
|
||||
id: `fake-stream-${Math.random().toString(36).slice(2)}`,
|
||||
getTracks: () => tracks,
|
||||
getVideoTracks: () => [videoTrack],
|
||||
getAudioTracks: () => (audioTrack ? [audioTrack] : []),
|
||||
addTrack: () => undefined,
|
||||
removeTrack: () => undefined,
|
||||
clone: () => createFakeMediaStream(withAudio),
|
||||
} as unknown as MediaStream;
|
||||
};
|
||||
|
||||
class FakePose {
|
||||
callback = null;
|
||||
|
||||
@@ -894,9 +961,19 @@ export async function installAppMocks(
|
||||
value: async () => undefined,
|
||||
});
|
||||
|
||||
Object.defineProperty(HTMLMediaElement.prototype, "srcObject", {
|
||||
configurable: true,
|
||||
get() {
|
||||
return (this as HTMLMediaElement & { __srcObject?: MediaStream }).__srcObject ?? null;
|
||||
},
|
||||
set(value) {
|
||||
(this as HTMLMediaElement & { __srcObject?: MediaStream }).__srcObject = value as MediaStream;
|
||||
},
|
||||
});
|
||||
|
||||
Object.defineProperty(HTMLCanvasElement.prototype, "captureStream", {
|
||||
configurable: true,
|
||||
value: () => new MediaStream(),
|
||||
value: () => createFakeMediaStream(),
|
||||
});
|
||||
|
||||
class FakeMediaRecorder extends EventTarget {
|
||||
@@ -961,7 +1038,7 @@ export async function installAppMocks(
|
||||
async setRemoteDescription(description: { type: string; sdp: string }) {
|
||||
this.remoteDescription = description;
|
||||
this.connectionState = "connected";
|
||||
this.ontrack?.({ streams: [new MediaStream()] });
|
||||
this.ontrack?.({ streams: [createFakeMediaStream()] });
|
||||
this.onconnectionstatechange?.();
|
||||
}
|
||||
|
||||
@@ -984,7 +1061,7 @@ export async function installAppMocks(
|
||||
Object.defineProperty(navigator, "mediaDevices", {
|
||||
configurable: true,
|
||||
value: {
|
||||
getUserMedia: async () => new MediaStream(),
|
||||
getUserMedia: async (constraints?: { audio?: unknown }) => createFakeMediaStream(Boolean(constraints?.audio)),
|
||||
enumerateDevices: async () => [
|
||||
{ deviceId: "cam-1", kind: "videoinput", label: "Front Camera", groupId: "g1" },
|
||||
{ deviceId: "cam-2", kind: "videoinput", label: "Back Camera", groupId: "g1" },
|
||||
|
||||
在新工单中引用
屏蔽一个用户