Fix live camera analysis loop
这个提交包含在:
@@ -33,6 +33,7 @@ export default function LiveCamera() {
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const poseRef = useRef<any>(null);
|
||||
const animFrameRef = useRef<number>(0);
|
||||
const analyzingRef = useRef(false);
|
||||
|
||||
const [cameraActive, setCameraActive] = useState(false);
|
||||
const [facing, setFacing] = useState<CameraFacing>("environment");
|
||||
@@ -108,6 +109,7 @@ export default function LiveCamera() {
|
||||
}, [facing]);
|
||||
|
||||
const stopCamera = useCallback(() => {
|
||||
analyzingRef.current = false;
|
||||
if (streamRef.current) {
|
||||
streamRef.current.getTracks().forEach(t => t.stop());
|
||||
streamRef.current = null;
|
||||
@@ -115,9 +117,12 @@ export default function LiveCamera() {
|
||||
if (animFrameRef.current) {
|
||||
cancelAnimationFrame(animFrameRef.current);
|
||||
}
|
||||
poseRef.current?.close?.();
|
||||
poseRef.current = null;
|
||||
setCameraActive(false);
|
||||
setAnalyzing(false);
|
||||
setLiveScore(null);
|
||||
setFeedback([]);
|
||||
}, []);
|
||||
|
||||
const switchCamera = useCallback(() => {
|
||||
@@ -131,17 +136,29 @@ export default function LiveCamera() {
|
||||
|
||||
// Start pose analysis
|
||||
const startAnalysis = useCallback(async () => {
|
||||
if (!videoRef.current || !canvasRef.current) return;
|
||||
if (!videoRef.current || !canvasRef.current || !cameraActive) {
|
||||
toast.error("请先启动摄像头");
|
||||
return;
|
||||
}
|
||||
if (analyzingRef.current) return;
|
||||
|
||||
analyzingRef.current = true;
|
||||
setAnalyzing(true);
|
||||
toast.info("正在加载姿势识别模型...");
|
||||
|
||||
try {
|
||||
const { Pose } = await import("@mediapipe/pose");
|
||||
const { drawConnectors, drawLandmarks } = await import("@mediapipe/drawing_utils");
|
||||
const testFactory = (
|
||||
window as typeof window & {
|
||||
__TEST_MEDIAPIPE_FACTORY__?: () => Promise<{ Pose: any }>;
|
||||
}
|
||||
).__TEST_MEDIAPIPE_FACTORY__;
|
||||
const { Pose } = testFactory
|
||||
? await testFactory()
|
||||
: await import("@mediapipe/pose");
|
||||
const allowSyntheticFrames = Boolean(testFactory);
|
||||
|
||||
const pose = new Pose({
|
||||
locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/pose/${file}`,
|
||||
locateFile: (file: string) => `https://cdn.jsdelivr.net/npm/@mediapipe/pose/${file}`,
|
||||
});
|
||||
|
||||
pose.setOptions({
|
||||
@@ -216,8 +233,8 @@ export default function LiveCamera() {
|
||||
poseRef.current = pose;
|
||||
|
||||
const processFrame = async () => {
|
||||
if (!videoRef.current || !analyzing) return;
|
||||
if (videoRef.current.readyState >= 2) {
|
||||
if (!videoRef.current || !analyzingRef.current) return;
|
||||
if (allowSyntheticFrames || videoRef.current.readyState >= 2) {
|
||||
await pose.send({ image: videoRef.current });
|
||||
}
|
||||
animFrameRef.current = requestAnimationFrame(processFrame);
|
||||
@@ -228,16 +245,21 @@ export default function LiveCamera() {
|
||||
} catch (err) {
|
||||
console.error("Pose init error:", err);
|
||||
toast.error("姿势识别模型加载失败");
|
||||
analyzingRef.current = false;
|
||||
setAnalyzing(false);
|
||||
}
|
||||
}, [analyzing, exerciseType]);
|
||||
}, [cameraActive, exerciseType]);
|
||||
|
||||
const stopAnalysis = useCallback(() => {
|
||||
analyzingRef.current = false;
|
||||
if (animFrameRef.current) {
|
||||
cancelAnimationFrame(animFrameRef.current);
|
||||
}
|
||||
poseRef.current?.close?.();
|
||||
poseRef.current = null;
|
||||
setAnalyzing(false);
|
||||
setLiveScore(null);
|
||||
setFeedback([]);
|
||||
}, []);
|
||||
|
||||
// Cleanup on unmount
|
||||
@@ -328,14 +350,14 @@ export default function LiveCamera() {
|
||||
<div className="relative bg-black aspect-video w-full">
|
||||
<video
|
||||
ref={videoRef}
|
||||
className={`absolute inset-0 w-full h-full object-contain ${analyzing ? "opacity-0" : ""}`}
|
||||
className="absolute inset-0 w-full h-full object-contain"
|
||||
playsInline
|
||||
muted
|
||||
autoPlay
|
||||
/>
|
||||
<canvas
|
||||
ref={canvasRef}
|
||||
className={`absolute inset-0 w-full h-full object-contain ${analyzing ? "" : "hidden"}`}
|
||||
className={`pointer-events-none absolute inset-0 w-full h-full object-contain ${analyzing ? "" : "hidden"}`}
|
||||
/>
|
||||
{!cameraActive && (
|
||||
<div className="absolute inset-0 flex flex-col items-center justify-center text-white/60">
|
||||
@@ -372,7 +394,7 @@ export default function LiveCamera() {
|
||||
</Button>
|
||||
)}
|
||||
{!analyzing ? (
|
||||
<Button size="sm" onClick={startAnalysis} className="gap-1.5">
|
||||
<Button data-testid="live-camera-analyze-button" size="sm" onClick={startAnalysis} className="gap-1.5">
|
||||
<Zap className="h-3.5 w-3.5" />开始分析
|
||||
</Button>
|
||||
) : (
|
||||
@@ -402,7 +424,7 @@ export default function LiveCamera() {
|
||||
{liveScore ? (
|
||||
<>
|
||||
<div className="text-center">
|
||||
<p className="text-4xl font-bold text-primary">{liveScore.overall}</p>
|
||||
<p data-testid="live-camera-score-overall" className="text-4xl font-bold text-primary">{liveScore.overall}</p>
|
||||
<p className="text-xs text-muted-foreground">综合评分</p>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
|
||||
@@ -40,6 +40,21 @@ test("live camera page exposes camera startup controls", async ({ page }) => {
|
||||
await expect(page.getByTestId("live-camera-start-button")).toBeVisible();
|
||||
});
|
||||
|
||||
test("live camera starts analysis and produces scores", async ({ page }) => {
|
||||
await installAppMocks(page, { authenticated: true });
|
||||
|
||||
await page.goto("/live-camera");
|
||||
await page.getByRole("button", { name: "下一步" }).click();
|
||||
await page.getByRole("button", { name: "下一步" }).click();
|
||||
await page.getByRole("button", { name: "下一步" }).click();
|
||||
await page.getByRole("button", { name: /启用摄像头/ }).click();
|
||||
|
||||
await expect(page.getByTestId("live-camera-analyze-button")).toBeVisible();
|
||||
await page.getByTestId("live-camera-analyze-button").click();
|
||||
|
||||
await expect(page.getByTestId("live-camera-score-overall")).toBeVisible();
|
||||
});
|
||||
|
||||
test("recorder flow archives a session and exposes it in videos", async ({ page }) => {
|
||||
await installAppMocks(page, { authenticated: true, videos: [] });
|
||||
|
||||
|
||||
@@ -322,6 +322,54 @@ export async function installAppMocks(
|
||||
};
|
||||
|
||||
await page.addInitScript(() => {
|
||||
const buildFakeLandmarks = () => {
|
||||
const points = Array.from({ length: 33 }, () => ({
|
||||
x: 0.5,
|
||||
y: 0.5,
|
||||
z: 0,
|
||||
visibility: 0.99,
|
||||
}));
|
||||
|
||||
points[0] = { x: 0.5, y: 0.15, z: 0, visibility: 0.99 };
|
||||
points[11] = { x: 0.42, y: 0.28, z: 0, visibility: 0.99 };
|
||||
points[12] = { x: 0.58, y: 0.28, z: 0, visibility: 0.99 };
|
||||
points[13] = { x: 0.36, y: 0.42, z: 0, visibility: 0.99 };
|
||||
points[14] = { x: 0.64, y: 0.42, z: 0, visibility: 0.99 };
|
||||
points[15] = { x: 0.3, y: 0.54, z: 0, visibility: 0.99 };
|
||||
points[16] = { x: 0.7, y: 0.52, z: 0, visibility: 0.99 };
|
||||
points[23] = { x: 0.45, y: 0.58, z: 0, visibility: 0.99 };
|
||||
points[24] = { x: 0.55, y: 0.58, z: 0, visibility: 0.99 };
|
||||
points[25] = { x: 0.44, y: 0.76, z: 0, visibility: 0.99 };
|
||||
points[26] = { x: 0.56, y: 0.76, z: 0, visibility: 0.99 };
|
||||
points[27] = { x: 0.42, y: 0.94, z: 0, visibility: 0.99 };
|
||||
points[28] = { x: 0.58, y: 0.94, z: 0, visibility: 0.99 };
|
||||
|
||||
return points;
|
||||
};
|
||||
|
||||
class FakePose {
|
||||
callback = null;
|
||||
|
||||
constructor(_config: unknown) {}
|
||||
|
||||
setOptions() {}
|
||||
|
||||
onResults(callback: (results: { poseLandmarks: ReturnType<typeof buildFakeLandmarks> }) => void) {
|
||||
this.callback = callback;
|
||||
}
|
||||
|
||||
async send() {
|
||||
this.callback?.({ poseLandmarks: buildFakeLandmarks() });
|
||||
}
|
||||
|
||||
close() {}
|
||||
}
|
||||
|
||||
Object.defineProperty(window, "__TEST_MEDIAPIPE_FACTORY__", {
|
||||
configurable: true,
|
||||
value: async () => ({ Pose: FakePose }),
|
||||
});
|
||||
|
||||
Object.defineProperty(HTMLMediaElement.prototype, "play", {
|
||||
configurable: true,
|
||||
value: async () => undefined,
|
||||
|
||||
在新工单中引用
屏蔽一个用户