Fix live camera analysis loop

这个提交包含在:
cryptocommuniums-afk
2026-03-14 22:54:15 +08:00
父节点 bc01a40564
当前提交 6943754838
修改 3 个文件,包含 96 行新增11 行删除

查看文件

@@ -33,6 +33,7 @@ export default function LiveCamera() {
const streamRef = useRef<MediaStream | null>(null); const streamRef = useRef<MediaStream | null>(null);
const poseRef = useRef<any>(null); const poseRef = useRef<any>(null);
const animFrameRef = useRef<number>(0); const animFrameRef = useRef<number>(0);
const analyzingRef = useRef(false);
const [cameraActive, setCameraActive] = useState(false); const [cameraActive, setCameraActive] = useState(false);
const [facing, setFacing] = useState<CameraFacing>("environment"); const [facing, setFacing] = useState<CameraFacing>("environment");
@@ -108,6 +109,7 @@ export default function LiveCamera() {
}, [facing]); }, [facing]);
const stopCamera = useCallback(() => { const stopCamera = useCallback(() => {
analyzingRef.current = false;
if (streamRef.current) { if (streamRef.current) {
streamRef.current.getTracks().forEach(t => t.stop()); streamRef.current.getTracks().forEach(t => t.stop());
streamRef.current = null; streamRef.current = null;
@@ -115,9 +117,12 @@ export default function LiveCamera() {
if (animFrameRef.current) { if (animFrameRef.current) {
cancelAnimationFrame(animFrameRef.current); cancelAnimationFrame(animFrameRef.current);
} }
poseRef.current?.close?.();
poseRef.current = null;
setCameraActive(false); setCameraActive(false);
setAnalyzing(false); setAnalyzing(false);
setLiveScore(null); setLiveScore(null);
setFeedback([]);
}, []); }, []);
const switchCamera = useCallback(() => { const switchCamera = useCallback(() => {
@@ -131,17 +136,29 @@ export default function LiveCamera() {
// Start pose analysis // Start pose analysis
const startAnalysis = useCallback(async () => { const startAnalysis = useCallback(async () => {
if (!videoRef.current || !canvasRef.current) return; if (!videoRef.current || !canvasRef.current || !cameraActive) {
toast.error("请先启动摄像头");
return;
}
if (analyzingRef.current) return;
analyzingRef.current = true;
setAnalyzing(true); setAnalyzing(true);
toast.info("正在加载姿势识别模型..."); toast.info("正在加载姿势识别模型...");
try { try {
const { Pose } = await import("@mediapipe/pose"); const testFactory = (
const { drawConnectors, drawLandmarks } = await import("@mediapipe/drawing_utils"); window as typeof window & {
__TEST_MEDIAPIPE_FACTORY__?: () => Promise<{ Pose: any }>;
}
).__TEST_MEDIAPIPE_FACTORY__;
const { Pose } = testFactory
? await testFactory()
: await import("@mediapipe/pose");
const allowSyntheticFrames = Boolean(testFactory);
const pose = new Pose({ const pose = new Pose({
locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/pose/${file}`, locateFile: (file: string) => `https://cdn.jsdelivr.net/npm/@mediapipe/pose/${file}`,
}); });
pose.setOptions({ pose.setOptions({
@@ -216,8 +233,8 @@ export default function LiveCamera() {
poseRef.current = pose; poseRef.current = pose;
const processFrame = async () => { const processFrame = async () => {
if (!videoRef.current || !analyzing) return; if (!videoRef.current || !analyzingRef.current) return;
if (videoRef.current.readyState >= 2) { if (allowSyntheticFrames || videoRef.current.readyState >= 2) {
await pose.send({ image: videoRef.current }); await pose.send({ image: videoRef.current });
} }
animFrameRef.current = requestAnimationFrame(processFrame); animFrameRef.current = requestAnimationFrame(processFrame);
@@ -228,16 +245,21 @@ export default function LiveCamera() {
} catch (err) { } catch (err) {
console.error("Pose init error:", err); console.error("Pose init error:", err);
toast.error("姿势识别模型加载失败"); toast.error("姿势识别模型加载失败");
analyzingRef.current = false;
setAnalyzing(false); setAnalyzing(false);
} }
}, [analyzing, exerciseType]); }, [cameraActive, exerciseType]);
const stopAnalysis = useCallback(() => { const stopAnalysis = useCallback(() => {
analyzingRef.current = false;
if (animFrameRef.current) { if (animFrameRef.current) {
cancelAnimationFrame(animFrameRef.current); cancelAnimationFrame(animFrameRef.current);
} }
poseRef.current?.close?.();
poseRef.current = null;
setAnalyzing(false); setAnalyzing(false);
setLiveScore(null); setLiveScore(null);
setFeedback([]);
}, []); }, []);
// Cleanup on unmount // Cleanup on unmount
@@ -328,14 +350,14 @@ export default function LiveCamera() {
<div className="relative bg-black aspect-video w-full"> <div className="relative bg-black aspect-video w-full">
<video <video
ref={videoRef} ref={videoRef}
className={`absolute inset-0 w-full h-full object-contain ${analyzing ? "opacity-0" : ""}`} className="absolute inset-0 w-full h-full object-contain"
playsInline playsInline
muted muted
autoPlay autoPlay
/> />
<canvas <canvas
ref={canvasRef} ref={canvasRef}
className={`absolute inset-0 w-full h-full object-contain ${analyzing ? "" : "hidden"}`} className={`pointer-events-none absolute inset-0 w-full h-full object-contain ${analyzing ? "" : "hidden"}`}
/> />
{!cameraActive && ( {!cameraActive && (
<div className="absolute inset-0 flex flex-col items-center justify-center text-white/60"> <div className="absolute inset-0 flex flex-col items-center justify-center text-white/60">
@@ -372,7 +394,7 @@ export default function LiveCamera() {
</Button> </Button>
)} )}
{!analyzing ? ( {!analyzing ? (
<Button size="sm" onClick={startAnalysis} className="gap-1.5"> <Button data-testid="live-camera-analyze-button" size="sm" onClick={startAnalysis} className="gap-1.5">
<Zap className="h-3.5 w-3.5" /> <Zap className="h-3.5 w-3.5" />
</Button> </Button>
) : ( ) : (
@@ -402,7 +424,7 @@ export default function LiveCamera() {
{liveScore ? ( {liveScore ? (
<> <>
<div className="text-center"> <div className="text-center">
<p className="text-4xl font-bold text-primary">{liveScore.overall}</p> <p data-testid="live-camera-score-overall" className="text-4xl font-bold text-primary">{liveScore.overall}</p>
<p className="text-xs text-muted-foreground"></p> <p className="text-xs text-muted-foreground"></p>
</div> </div>
<div className="space-y-2"> <div className="space-y-2">

查看文件

@@ -40,6 +40,21 @@ test("live camera page exposes camera startup controls", async ({ page }) => {
await expect(page.getByTestId("live-camera-start-button")).toBeVisible(); await expect(page.getByTestId("live-camera-start-button")).toBeVisible();
}); });
test("live camera starts analysis and produces scores", async ({ page }) => {
await installAppMocks(page, { authenticated: true });
await page.goto("/live-camera");
await page.getByRole("button", { name: "下一步" }).click();
await page.getByRole("button", { name: "下一步" }).click();
await page.getByRole("button", { name: "下一步" }).click();
await page.getByRole("button", { name: /启用摄像头/ }).click();
await expect(page.getByTestId("live-camera-analyze-button")).toBeVisible();
await page.getByTestId("live-camera-analyze-button").click();
await expect(page.getByTestId("live-camera-score-overall")).toBeVisible();
});
test("recorder flow archives a session and exposes it in videos", async ({ page }) => { test("recorder flow archives a session and exposes it in videos", async ({ page }) => {
await installAppMocks(page, { authenticated: true, videos: [] }); await installAppMocks(page, { authenticated: true, videos: [] });

查看文件

@@ -322,6 +322,54 @@ export async function installAppMocks(
}; };
await page.addInitScript(() => { await page.addInitScript(() => {
const buildFakeLandmarks = () => {
const points = Array.from({ length: 33 }, () => ({
x: 0.5,
y: 0.5,
z: 0,
visibility: 0.99,
}));
points[0] = { x: 0.5, y: 0.15, z: 0, visibility: 0.99 };
points[11] = { x: 0.42, y: 0.28, z: 0, visibility: 0.99 };
points[12] = { x: 0.58, y: 0.28, z: 0, visibility: 0.99 };
points[13] = { x: 0.36, y: 0.42, z: 0, visibility: 0.99 };
points[14] = { x: 0.64, y: 0.42, z: 0, visibility: 0.99 };
points[15] = { x: 0.3, y: 0.54, z: 0, visibility: 0.99 };
points[16] = { x: 0.7, y: 0.52, z: 0, visibility: 0.99 };
points[23] = { x: 0.45, y: 0.58, z: 0, visibility: 0.99 };
points[24] = { x: 0.55, y: 0.58, z: 0, visibility: 0.99 };
points[25] = { x: 0.44, y: 0.76, z: 0, visibility: 0.99 };
points[26] = { x: 0.56, y: 0.76, z: 0, visibility: 0.99 };
points[27] = { x: 0.42, y: 0.94, z: 0, visibility: 0.99 };
points[28] = { x: 0.58, y: 0.94, z: 0, visibility: 0.99 };
return points;
};
class FakePose {
callback = null;
constructor(_config: unknown) {}
setOptions() {}
onResults(callback: (results: { poseLandmarks: ReturnType<typeof buildFakeLandmarks> }) => void) {
this.callback = callback;
}
async send() {
this.callback?.({ poseLandmarks: buildFakeLandmarks() });
}
close() {}
}
Object.defineProperty(window, "__TEST_MEDIAPIPE_FACTORY__", {
configurable: true,
value: async () => ({ Pose: FakePose }),
});
Object.defineProperty(HTMLMediaElement.prototype, "play", { Object.defineProperty(HTMLMediaElement.prototype, "play", {
configurable: true, configurable: true,
value: async () => undefined, value: async () => undefined,