Fix live camera analysis loop
这个提交包含在:
@@ -33,6 +33,7 @@ export default function LiveCamera() {
|
|||||||
const streamRef = useRef<MediaStream | null>(null);
|
const streamRef = useRef<MediaStream | null>(null);
|
||||||
const poseRef = useRef<any>(null);
|
const poseRef = useRef<any>(null);
|
||||||
const animFrameRef = useRef<number>(0);
|
const animFrameRef = useRef<number>(0);
|
||||||
|
const analyzingRef = useRef(false);
|
||||||
|
|
||||||
const [cameraActive, setCameraActive] = useState(false);
|
const [cameraActive, setCameraActive] = useState(false);
|
||||||
const [facing, setFacing] = useState<CameraFacing>("environment");
|
const [facing, setFacing] = useState<CameraFacing>("environment");
|
||||||
@@ -108,6 +109,7 @@ export default function LiveCamera() {
|
|||||||
}, [facing]);
|
}, [facing]);
|
||||||
|
|
||||||
const stopCamera = useCallback(() => {
|
const stopCamera = useCallback(() => {
|
||||||
|
analyzingRef.current = false;
|
||||||
if (streamRef.current) {
|
if (streamRef.current) {
|
||||||
streamRef.current.getTracks().forEach(t => t.stop());
|
streamRef.current.getTracks().forEach(t => t.stop());
|
||||||
streamRef.current = null;
|
streamRef.current = null;
|
||||||
@@ -115,9 +117,12 @@ export default function LiveCamera() {
|
|||||||
if (animFrameRef.current) {
|
if (animFrameRef.current) {
|
||||||
cancelAnimationFrame(animFrameRef.current);
|
cancelAnimationFrame(animFrameRef.current);
|
||||||
}
|
}
|
||||||
|
poseRef.current?.close?.();
|
||||||
|
poseRef.current = null;
|
||||||
setCameraActive(false);
|
setCameraActive(false);
|
||||||
setAnalyzing(false);
|
setAnalyzing(false);
|
||||||
setLiveScore(null);
|
setLiveScore(null);
|
||||||
|
setFeedback([]);
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const switchCamera = useCallback(() => {
|
const switchCamera = useCallback(() => {
|
||||||
@@ -131,17 +136,29 @@ export default function LiveCamera() {
|
|||||||
|
|
||||||
// Start pose analysis
|
// Start pose analysis
|
||||||
const startAnalysis = useCallback(async () => {
|
const startAnalysis = useCallback(async () => {
|
||||||
if (!videoRef.current || !canvasRef.current) return;
|
if (!videoRef.current || !canvasRef.current || !cameraActive) {
|
||||||
|
toast.error("请先启动摄像头");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (analyzingRef.current) return;
|
||||||
|
|
||||||
|
analyzingRef.current = true;
|
||||||
setAnalyzing(true);
|
setAnalyzing(true);
|
||||||
toast.info("正在加载姿势识别模型...");
|
toast.info("正在加载姿势识别模型...");
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const { Pose } = await import("@mediapipe/pose");
|
const testFactory = (
|
||||||
const { drawConnectors, drawLandmarks } = await import("@mediapipe/drawing_utils");
|
window as typeof window & {
|
||||||
|
__TEST_MEDIAPIPE_FACTORY__?: () => Promise<{ Pose: any }>;
|
||||||
|
}
|
||||||
|
).__TEST_MEDIAPIPE_FACTORY__;
|
||||||
|
const { Pose } = testFactory
|
||||||
|
? await testFactory()
|
||||||
|
: await import("@mediapipe/pose");
|
||||||
|
const allowSyntheticFrames = Boolean(testFactory);
|
||||||
|
|
||||||
const pose = new Pose({
|
const pose = new Pose({
|
||||||
locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/pose/${file}`,
|
locateFile: (file: string) => `https://cdn.jsdelivr.net/npm/@mediapipe/pose/${file}`,
|
||||||
});
|
});
|
||||||
|
|
||||||
pose.setOptions({
|
pose.setOptions({
|
||||||
@@ -216,8 +233,8 @@ export default function LiveCamera() {
|
|||||||
poseRef.current = pose;
|
poseRef.current = pose;
|
||||||
|
|
||||||
const processFrame = async () => {
|
const processFrame = async () => {
|
||||||
if (!videoRef.current || !analyzing) return;
|
if (!videoRef.current || !analyzingRef.current) return;
|
||||||
if (videoRef.current.readyState >= 2) {
|
if (allowSyntheticFrames || videoRef.current.readyState >= 2) {
|
||||||
await pose.send({ image: videoRef.current });
|
await pose.send({ image: videoRef.current });
|
||||||
}
|
}
|
||||||
animFrameRef.current = requestAnimationFrame(processFrame);
|
animFrameRef.current = requestAnimationFrame(processFrame);
|
||||||
@@ -228,16 +245,21 @@ export default function LiveCamera() {
|
|||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error("Pose init error:", err);
|
console.error("Pose init error:", err);
|
||||||
toast.error("姿势识别模型加载失败");
|
toast.error("姿势识别模型加载失败");
|
||||||
|
analyzingRef.current = false;
|
||||||
setAnalyzing(false);
|
setAnalyzing(false);
|
||||||
}
|
}
|
||||||
}, [analyzing, exerciseType]);
|
}, [cameraActive, exerciseType]);
|
||||||
|
|
||||||
const stopAnalysis = useCallback(() => {
|
const stopAnalysis = useCallback(() => {
|
||||||
|
analyzingRef.current = false;
|
||||||
if (animFrameRef.current) {
|
if (animFrameRef.current) {
|
||||||
cancelAnimationFrame(animFrameRef.current);
|
cancelAnimationFrame(animFrameRef.current);
|
||||||
}
|
}
|
||||||
|
poseRef.current?.close?.();
|
||||||
|
poseRef.current = null;
|
||||||
setAnalyzing(false);
|
setAnalyzing(false);
|
||||||
setLiveScore(null);
|
setLiveScore(null);
|
||||||
|
setFeedback([]);
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
// Cleanup on unmount
|
// Cleanup on unmount
|
||||||
@@ -328,14 +350,14 @@ export default function LiveCamera() {
|
|||||||
<div className="relative bg-black aspect-video w-full">
|
<div className="relative bg-black aspect-video w-full">
|
||||||
<video
|
<video
|
||||||
ref={videoRef}
|
ref={videoRef}
|
||||||
className={`absolute inset-0 w-full h-full object-contain ${analyzing ? "opacity-0" : ""}`}
|
className="absolute inset-0 w-full h-full object-contain"
|
||||||
playsInline
|
playsInline
|
||||||
muted
|
muted
|
||||||
autoPlay
|
autoPlay
|
||||||
/>
|
/>
|
||||||
<canvas
|
<canvas
|
||||||
ref={canvasRef}
|
ref={canvasRef}
|
||||||
className={`absolute inset-0 w-full h-full object-contain ${analyzing ? "" : "hidden"}`}
|
className={`pointer-events-none absolute inset-0 w-full h-full object-contain ${analyzing ? "" : "hidden"}`}
|
||||||
/>
|
/>
|
||||||
{!cameraActive && (
|
{!cameraActive && (
|
||||||
<div className="absolute inset-0 flex flex-col items-center justify-center text-white/60">
|
<div className="absolute inset-0 flex flex-col items-center justify-center text-white/60">
|
||||||
@@ -372,7 +394,7 @@ export default function LiveCamera() {
|
|||||||
</Button>
|
</Button>
|
||||||
)}
|
)}
|
||||||
{!analyzing ? (
|
{!analyzing ? (
|
||||||
<Button size="sm" onClick={startAnalysis} className="gap-1.5">
|
<Button data-testid="live-camera-analyze-button" size="sm" onClick={startAnalysis} className="gap-1.5">
|
||||||
<Zap className="h-3.5 w-3.5" />开始分析
|
<Zap className="h-3.5 w-3.5" />开始分析
|
||||||
</Button>
|
</Button>
|
||||||
) : (
|
) : (
|
||||||
@@ -402,7 +424,7 @@ export default function LiveCamera() {
|
|||||||
{liveScore ? (
|
{liveScore ? (
|
||||||
<>
|
<>
|
||||||
<div className="text-center">
|
<div className="text-center">
|
||||||
<p className="text-4xl font-bold text-primary">{liveScore.overall}</p>
|
<p data-testid="live-camera-score-overall" className="text-4xl font-bold text-primary">{liveScore.overall}</p>
|
||||||
<p className="text-xs text-muted-foreground">综合评分</p>
|
<p className="text-xs text-muted-foreground">综合评分</p>
|
||||||
</div>
|
</div>
|
||||||
<div className="space-y-2">
|
<div className="space-y-2">
|
||||||
|
|||||||
@@ -40,6 +40,21 @@ test("live camera page exposes camera startup controls", async ({ page }) => {
|
|||||||
await expect(page.getByTestId("live-camera-start-button")).toBeVisible();
|
await expect(page.getByTestId("live-camera-start-button")).toBeVisible();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("live camera starts analysis and produces scores", async ({ page }) => {
|
||||||
|
await installAppMocks(page, { authenticated: true });
|
||||||
|
|
||||||
|
await page.goto("/live-camera");
|
||||||
|
await page.getByRole("button", { name: "下一步" }).click();
|
||||||
|
await page.getByRole("button", { name: "下一步" }).click();
|
||||||
|
await page.getByRole("button", { name: "下一步" }).click();
|
||||||
|
await page.getByRole("button", { name: /启用摄像头/ }).click();
|
||||||
|
|
||||||
|
await expect(page.getByTestId("live-camera-analyze-button")).toBeVisible();
|
||||||
|
await page.getByTestId("live-camera-analyze-button").click();
|
||||||
|
|
||||||
|
await expect(page.getByTestId("live-camera-score-overall")).toBeVisible();
|
||||||
|
});
|
||||||
|
|
||||||
test("recorder flow archives a session and exposes it in videos", async ({ page }) => {
|
test("recorder flow archives a session and exposes it in videos", async ({ page }) => {
|
||||||
await installAppMocks(page, { authenticated: true, videos: [] });
|
await installAppMocks(page, { authenticated: true, videos: [] });
|
||||||
|
|
||||||
|
|||||||
@@ -322,6 +322,54 @@ export async function installAppMocks(
|
|||||||
};
|
};
|
||||||
|
|
||||||
await page.addInitScript(() => {
|
await page.addInitScript(() => {
|
||||||
|
const buildFakeLandmarks = () => {
|
||||||
|
const points = Array.from({ length: 33 }, () => ({
|
||||||
|
x: 0.5,
|
||||||
|
y: 0.5,
|
||||||
|
z: 0,
|
||||||
|
visibility: 0.99,
|
||||||
|
}));
|
||||||
|
|
||||||
|
points[0] = { x: 0.5, y: 0.15, z: 0, visibility: 0.99 };
|
||||||
|
points[11] = { x: 0.42, y: 0.28, z: 0, visibility: 0.99 };
|
||||||
|
points[12] = { x: 0.58, y: 0.28, z: 0, visibility: 0.99 };
|
||||||
|
points[13] = { x: 0.36, y: 0.42, z: 0, visibility: 0.99 };
|
||||||
|
points[14] = { x: 0.64, y: 0.42, z: 0, visibility: 0.99 };
|
||||||
|
points[15] = { x: 0.3, y: 0.54, z: 0, visibility: 0.99 };
|
||||||
|
points[16] = { x: 0.7, y: 0.52, z: 0, visibility: 0.99 };
|
||||||
|
points[23] = { x: 0.45, y: 0.58, z: 0, visibility: 0.99 };
|
||||||
|
points[24] = { x: 0.55, y: 0.58, z: 0, visibility: 0.99 };
|
||||||
|
points[25] = { x: 0.44, y: 0.76, z: 0, visibility: 0.99 };
|
||||||
|
points[26] = { x: 0.56, y: 0.76, z: 0, visibility: 0.99 };
|
||||||
|
points[27] = { x: 0.42, y: 0.94, z: 0, visibility: 0.99 };
|
||||||
|
points[28] = { x: 0.58, y: 0.94, z: 0, visibility: 0.99 };
|
||||||
|
|
||||||
|
return points;
|
||||||
|
};
|
||||||
|
|
||||||
|
class FakePose {
|
||||||
|
callback = null;
|
||||||
|
|
||||||
|
constructor(_config: unknown) {}
|
||||||
|
|
||||||
|
setOptions() {}
|
||||||
|
|
||||||
|
onResults(callback: (results: { poseLandmarks: ReturnType<typeof buildFakeLandmarks> }) => void) {
|
||||||
|
this.callback = callback;
|
||||||
|
}
|
||||||
|
|
||||||
|
async send() {
|
||||||
|
this.callback?.({ poseLandmarks: buildFakeLandmarks() });
|
||||||
|
}
|
||||||
|
|
||||||
|
close() {}
|
||||||
|
}
|
||||||
|
|
||||||
|
Object.defineProperty(window, "__TEST_MEDIAPIPE_FACTORY__", {
|
||||||
|
configurable: true,
|
||||||
|
value: async () => ({ Pose: FakePose }),
|
||||||
|
});
|
||||||
|
|
||||||
Object.defineProperty(HTMLMediaElement.prototype, "play", {
|
Object.defineProperty(HTMLMediaElement.prototype, "play", {
|
||||||
configurable: true,
|
configurable: true,
|
||||||
value: async () => undefined,
|
value: async () => undefined,
|
||||||
|
|||||||
在新工单中引用
屏蔽一个用户