文件
tennis-training-hub/client/src/pages/LiveCamera.tsx

573 行
22 KiB
TypeScript
原始文件 Blame 文件历史

此文件含有模棱两可的 Unicode 字符
此文件含有可能会与其他字符混淆的 Unicode 字符。 如果您是想特意这样的,可以安全地忽略该警告。 使用 Escape 按钮显示他们。
import { useAuth } from "@/_core/hooks/useAuth";
import { trpc } from "@/lib/trpc";
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
import { Button } from "@/components/ui/button";
import { Badge } from "@/components/ui/badge";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogFooter, DialogDescription } from "@/components/ui/dialog";
import { toast } from "sonner";
import { Camera, CameraOff, RotateCcw, CheckCircle2, AlertTriangle, Smartphone, Monitor, FlipHorizontal, Zap, Activity } from "lucide-react";
import { useRef, useState, useEffect, useCallback } from "react";
type CameraFacing = "user" | "environment";
interface PoseScore {
overall: number;
posture: number;
balance: number;
armAngle: number;
}
// Camera position guide steps
const SETUP_STEPS = [
{ title: "放置设备", desc: "将手机/平板固定在三脚架或稳定平面上", icon: <Smartphone className="h-6 w-6" /> },
{ title: "调整距离", desc: "确保摄像头能拍到全身距离2-3米", icon: <Monitor className="h-6 w-6" /> },
{ title: "调整高度", desc: "摄像头高度约在腰部位置,略微仰角", icon: <Camera className="h-6 w-6" /> },
{ title: "确认画面", desc: "确保光线充足,背景简洁,全身可见", icon: <CheckCircle2 className="h-6 w-6" /> },
];
export default function LiveCamera() {
const { user } = useAuth();
const videoRef = useRef<HTMLVideoElement>(null);
const canvasRef = useRef<HTMLCanvasElement>(null);
const streamRef = useRef<MediaStream | null>(null);
const poseRef = useRef<any>(null);
const animFrameRef = useRef<number>(0);
const [cameraActive, setCameraActive] = useState(false);
const [facing, setFacing] = useState<CameraFacing>("environment");
const [showSetupGuide, setShowSetupGuide] = useState(true);
const [setupStep, setSetupStep] = useState(0);
const [analyzing, setAnalyzing] = useState(false);
const [liveScore, setLiveScore] = useState<PoseScore | null>(null);
const [frameCount, setFrameCount] = useState(0);
const [fps, setFps] = useState(0);
const [hasMultipleCameras, setHasMultipleCameras] = useState(false);
const [exerciseType, setExerciseType] = useState("forehand");
const [feedback, setFeedback] = useState<string[]>([]);
// Check available cameras
useEffect(() => {
navigator.mediaDevices?.enumerateDevices().then(devices => {
const cameras = devices.filter(d => d.kind === "videoinput");
setHasMultipleCameras(cameras.length > 1);
}).catch(() => {});
}, []);
// FPS counter
useEffect(() => {
if (!analyzing) return;
const interval = setInterval(() => {
setFps(prev => {
const current = frameCount;
setFrameCount(0);
return current;
});
}, 1000);
return () => clearInterval(interval);
}, [analyzing, frameCount]);
const startCamera = useCallback(async () => {
try {
// Stop existing stream
if (streamRef.current) {
streamRef.current.getTracks().forEach(t => t.stop());
}
const isMobile = /Android|iPhone|iPad/i.test(navigator.userAgent);
const constraints: MediaStreamConstraints = {
video: {
facingMode: facing,
width: { ideal: isMobile ? 1280 : 1920 },
height: { ideal: isMobile ? 720 : 1080 },
frameRate: { ideal: 30, max: 30 },
},
audio: false,
};
const stream = await navigator.mediaDevices.getUserMedia(constraints);
streamRef.current = stream;
if (videoRef.current) {
videoRef.current.srcObject = stream;
await videoRef.current.play();
}
setCameraActive(true);
toast.success("摄像头已启动");
} catch (err: any) {
console.error("Camera error:", err);
if (err.name === "NotAllowedError") {
toast.error("请允许摄像头访问权限");
} else if (err.name === "NotFoundError") {
toast.error("未找到摄像头设备");
} else {
toast.error("摄像头启动失败: " + err.message);
}
}
}, [facing]);
const stopCamera = useCallback(() => {
if (streamRef.current) {
streamRef.current.getTracks().forEach(t => t.stop());
streamRef.current = null;
}
if (animFrameRef.current) {
cancelAnimationFrame(animFrameRef.current);
}
setCameraActive(false);
setAnalyzing(false);
setLiveScore(null);
}, []);
const switchCamera = useCallback(() => {
const newFacing = facing === "user" ? "environment" : "user";
setFacing(newFacing);
if (cameraActive) {
stopCamera();
setTimeout(() => startCamera(), 300);
}
}, [facing, cameraActive, stopCamera, startCamera]);
// Start pose analysis
const startAnalysis = useCallback(async () => {
if (!videoRef.current || !canvasRef.current) return;
setAnalyzing(true);
toast.info("正在加载姿势识别模型...");
try {
const { Pose } = await import("@mediapipe/pose");
const { drawConnectors, drawLandmarks } = await import("@mediapipe/drawing_utils");
const pose = new Pose({
locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/pose/${file}`,
});
pose.setOptions({
modelComplexity: 1,
smoothLandmarks: true,
enableSegmentation: false,
minDetectionConfidence: 0.5,
minTrackingConfidence: 0.5,
});
const POSE_CONNECTIONS = [
[11, 12], [11, 13], [13, 15], [12, 14], [14, 16],
[11, 23], [12, 24], [23, 24], [23, 25], [24, 26],
[25, 27], [26, 28], [15, 17], [16, 18], [15, 19],
[16, 20], [17, 19], [18, 20],
];
pose.onResults((results: any) => {
const canvas = canvasRef.current;
const ctx = canvas?.getContext("2d");
if (!canvas || !ctx || !videoRef.current) return;
canvas.width = videoRef.current.videoWidth;
canvas.height = videoRef.current.videoHeight;
ctx.clearRect(0, 0, canvas.width, canvas.height);
ctx.drawImage(videoRef.current, 0, 0);
if (results.poseLandmarks) {
// Draw skeleton
const landmarks = results.poseLandmarks;
// Draw connections
ctx.strokeStyle = "rgba(0, 200, 100, 0.8)";
ctx.lineWidth = 3;
POSE_CONNECTIONS.forEach(([a, b]) => {
const la = landmarks[a];
const lb = landmarks[b];
if (la && lb && la.visibility > 0.3 && lb.visibility > 0.3) {
ctx.beginPath();
ctx.moveTo(la.x * canvas.width, la.y * canvas.height);
ctx.lineTo(lb.x * canvas.width, lb.y * canvas.height);
ctx.stroke();
}
});
// Draw landmarks
landmarks.forEach((lm: any, i: number) => {
if (lm.visibility > 0.3) {
ctx.fillStyle = [11, 12, 13, 14, 15, 16].includes(i) ? "#ff4444" : "#00cc66";
ctx.beginPath();
ctx.arc(lm.x * canvas.width, lm.y * canvas.height, 5, 0, 2 * Math.PI);
ctx.fill();
}
});
// Calculate live scores
const score = calculateLiveScore(landmarks, exerciseType);
setLiveScore(score);
setFeedback(generateLiveFeedback(landmarks, exerciseType));
setFrameCount(prev => prev + 1);
// Draw score overlay
ctx.fillStyle = "rgba(0,0,0,0.6)";
ctx.fillRect(10, 10, 180, 40);
ctx.fillStyle = "#fff";
ctx.font = "bold 16px sans-serif";
ctx.fillText(`评分: ${score.overall}/100`, 20, 35);
}
});
poseRef.current = pose;
const processFrame = async () => {
if (!videoRef.current || !analyzing) return;
if (videoRef.current.readyState >= 2) {
await pose.send({ image: videoRef.current });
}
animFrameRef.current = requestAnimationFrame(processFrame);
};
toast.success("模型加载完成,开始实时分析");
processFrame();
} catch (err) {
console.error("Pose init error:", err);
toast.error("姿势识别模型加载失败");
setAnalyzing(false);
}
}, [analyzing, exerciseType]);
const stopAnalysis = useCallback(() => {
if (animFrameRef.current) {
cancelAnimationFrame(animFrameRef.current);
}
setAnalyzing(false);
setLiveScore(null);
}, []);
// Cleanup on unmount
useEffect(() => {
return () => {
stopCamera();
};
}, [stopCamera]);
// Setup guide dialog
const handleSetupComplete = () => {
setShowSetupGuide(false);
startCamera();
};
return (
<div className="space-y-4">
<div className="flex flex-col sm:flex-row sm:items-center sm:justify-between gap-2">
<div>
<h1 className="text-2xl font-bold tracking-tight">姿</h1>
<p className="text-muted-foreground text-sm mt-1"></p>
</div>
<div className="flex items-center gap-2">
<Select value={exerciseType} onValueChange={setExerciseType}>
<SelectTrigger className="w-[120px] h-9 text-sm">
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="forehand"></SelectItem>
<SelectItem value="backhand"></SelectItem>
<SelectItem value="serve"></SelectItem>
<SelectItem value="volley"></SelectItem>
<SelectItem value="footwork"></SelectItem>
</SelectContent>
</Select>
</div>
</div>
{/* Camera position setup guide */}
<Dialog open={showSetupGuide} onOpenChange={setShowSetupGuide}>
<DialogContent className="max-w-md">
<DialogHeader>
<DialogTitle className="flex items-center gap-2">
<Camera className="h-5 w-5 text-primary" />
</DialogTitle>
<DialogDescription>
</DialogDescription>
</DialogHeader>
<div className="space-y-4 py-2">
{SETUP_STEPS.map((step, idx) => (
<div key={idx} className={`flex items-start gap-3 p-3 rounded-lg transition-colors ${
idx === setupStep ? "bg-primary/10 border border-primary/20" : idx < setupStep ? "bg-green-50" : "bg-muted/30"
}`}>
<div className={`h-10 w-10 rounded-full flex items-center justify-center shrink-0 ${
idx < setupStep ? "bg-green-100 text-green-600" : idx === setupStep ? "bg-primary/20 text-primary" : "bg-muted text-muted-foreground"
}`}>
{idx < setupStep ? <CheckCircle2 className="h-5 w-5" /> : step.icon}
</div>
<div>
<p className="font-medium text-sm">{step.title}</p>
<p className="text-xs text-muted-foreground mt-0.5">{step.desc}</p>
</div>
</div>
))}
</div>
<DialogFooter className="flex gap-2">
{setupStep > 0 && (
<Button variant="outline" onClick={() => setSetupStep(s => s - 1)}></Button>
)}
{setupStep < SETUP_STEPS.length - 1 ? (
<Button onClick={() => setSetupStep(s => s + 1)}></Button>
) : (
<Button onClick={handleSetupComplete} className="gap-2">
<Camera className="h-4 w-4" />使
</Button>
)}
</DialogFooter>
</DialogContent>
</Dialog>
{/* Main camera view */}
<div className="grid grid-cols-1 lg:grid-cols-4 gap-4">
<div className="lg:col-span-3">
<Card className="border-0 shadow-sm overflow-hidden">
<CardContent className="p-0 relative">
<div className="relative bg-black aspect-video w-full">
<video
ref={videoRef}
className={`absolute inset-0 w-full h-full object-contain ${analyzing ? "opacity-0" : ""}`}
playsInline
muted
autoPlay
/>
<canvas
ref={canvasRef}
className={`absolute inset-0 w-full h-full object-contain ${analyzing ? "" : "hidden"}`}
/>
{!cameraActive && (
<div className="absolute inset-0 flex flex-col items-center justify-center text-white/60">
<CameraOff className="h-12 w-12 mb-3" />
<p className="text-sm"></p>
<Button variant="secondary" className="mt-3 gap-2" onClick={() => setShowSetupGuide(true)}>
<Camera className="h-4 w-4" />
</Button>
</div>
)}
{/* FPS indicator */}
{analyzing && (
<div className="absolute top-3 right-3 bg-black/60 text-white text-xs px-2 py-1 rounded">
{fps} FPS
</div>
)}
</div>
{/* Controls bar */}
<div className="flex items-center justify-center gap-3 p-3 bg-muted/30 flex-wrap">
{!cameraActive ? (
<Button onClick={() => setShowSetupGuide(true)} className="gap-2">
<Camera className="h-4 w-4" />
</Button>
) : (
<>
<Button variant="outline" size="sm" onClick={stopCamera} className="gap-1.5">
<CameraOff className="h-3.5 w-3.5" />
</Button>
{hasMultipleCameras && (
<Button variant="outline" size="sm" onClick={switchCamera} className="gap-1.5">
<FlipHorizontal className="h-3.5 w-3.5" />
{facing === "user" ? "后置" : "前置"}
</Button>
)}
{!analyzing ? (
<Button size="sm" onClick={startAnalysis} className="gap-1.5">
<Zap className="h-3.5 w-3.5" />
</Button>
) : (
<Button variant="destructive" size="sm" onClick={stopAnalysis} className="gap-1.5">
<Activity className="h-3.5 w-3.5" />
</Button>
)}
<Button variant="outline" size="sm" onClick={() => setShowSetupGuide(true)} className="gap-1.5">
<RotateCcw className="h-3.5 w-3.5" />
</Button>
</>
)}
</div>
</CardContent>
</Card>
</div>
{/* Live score panel */}
<div className="space-y-4">
<Card className="border-0 shadow-sm">
<CardHeader className="pb-2">
<CardTitle className="text-sm flex items-center gap-2">
<Activity className="h-4 w-4 text-primary" />
</CardTitle>
</CardHeader>
<CardContent className="space-y-3">
{liveScore ? (
<>
<div className="text-center">
<p className="text-4xl font-bold text-primary">{liveScore.overall}</p>
<p className="text-xs text-muted-foreground"></p>
</div>
<div className="space-y-2">
<ScoreBar label="姿势" value={liveScore.posture} />
<ScoreBar label="平衡" value={liveScore.balance} />
<ScoreBar label="手臂" value={liveScore.armAngle} />
</div>
</>
) : (
<div className="text-center py-6 text-muted-foreground text-sm">
<Activity className="h-8 w-8 mx-auto mb-2 opacity-30" />
<p></p>
</div>
)}
</CardContent>
</Card>
<Card className="border-0 shadow-sm">
<CardHeader className="pb-2">
<CardTitle className="text-sm flex items-center gap-2">
<AlertTriangle className="h-4 w-4 text-orange-500" />
</CardTitle>
</CardHeader>
<CardContent>
{feedback.length > 0 ? (
<div className="space-y-2">
{feedback.map((f, i) => (
<div key={i} className="text-xs p-2 rounded bg-orange-50 text-orange-700 border border-orange-100">
{f}
</div>
))}
</div>
) : (
<p className="text-xs text-muted-foreground text-center py-4">
</p>
)}
</CardContent>
</Card>
{/* Tips card */}
<Card className="border-0 shadow-sm bg-blue-50/50">
<CardContent className="py-3">
<p className="text-xs font-medium text-blue-700 mb-1">📱 </p>
<ul className="text-[11px] text-blue-600 space-y-1">
<li>· </li>
<li>· 使</li>
<li>· 2-3</li>
<li>· 线</li>
</ul>
</CardContent>
</Card>
</div>
</div>
</div>
);
}
// Score bar component
function ScoreBar({ label, value }: { label: string; value: number }) {
const color = value >= 80 ? "bg-green-500" : value >= 60 ? "bg-yellow-500" : "bg-red-500";
return (
<div>
<div className="flex justify-between text-xs mb-0.5">
<span className="text-muted-foreground">{label}</span>
<span className="font-medium">{Math.round(value)}</span>
</div>
<div className="h-1.5 bg-muted rounded-full overflow-hidden">
<div className={`h-full ${color} rounded-full transition-all duration-300`} style={{ width: `${value}%` }} />
</div>
</div>
);
}
// Live score calculation from landmarks
function calculateLiveScore(landmarks: any[], exerciseType: string): PoseScore {
const getAngle = (a: any, b: any, c: any) => {
const radians = Math.atan2(c.y - b.y, c.x - b.x) - Math.atan2(a.y - b.y, a.x - b.x);
let angle = Math.abs(radians * 180 / Math.PI);
if (angle > 180) angle = 360 - angle;
return angle;
};
// Shoulder alignment (posture)
const leftShoulder = landmarks[11];
const rightShoulder = landmarks[12];
const shoulderDiff = Math.abs(leftShoulder.y - rightShoulder.y);
const postureScore = Math.max(0, 100 - shoulderDiff * 500);
// Hip alignment (balance)
const leftHip = landmarks[23];
const rightHip = landmarks[24];
const hipDiff = Math.abs(leftHip.y - rightHip.y);
const balanceScore = Math.max(0, 100 - hipDiff * 500);
// Arm angle based on exercise type
let armScore = 70;
if (exerciseType === "forehand" || exerciseType === "backhand") {
const shoulder = exerciseType === "forehand" ? landmarks[12] : landmarks[11];
const elbow = exerciseType === "forehand" ? landmarks[14] : landmarks[13];
const wrist = exerciseType === "forehand" ? landmarks[16] : landmarks[15];
const angle = getAngle(shoulder, elbow, wrist);
// Ideal forehand/backhand elbow angle: 90-150 degrees
if (angle >= 90 && angle <= 150) armScore = 90 + (1 - Math.abs(angle - 120) / 30) * 10;
else armScore = Math.max(30, 90 - Math.abs(angle - 120));
} else if (exerciseType === "serve") {
const rightElbow = landmarks[14];
const rightShoulder2 = landmarks[12];
const rightWrist = landmarks[16];
const angle = getAngle(rightShoulder2, rightElbow, rightWrist);
if (angle >= 150 && angle <= 180) armScore = 95;
else armScore = Math.max(40, 95 - Math.abs(angle - 165) * 2);
}
const overall = Math.round(postureScore * 0.3 + balanceScore * 0.3 + armScore * 0.4);
return {
overall: Math.min(100, Math.max(0, overall)),
posture: Math.min(100, Math.max(0, Math.round(postureScore))),
balance: Math.min(100, Math.max(0, Math.round(balanceScore))),
armAngle: Math.min(100, Math.max(0, Math.round(armScore))),
};
}
// Generate live feedback
function generateLiveFeedback(landmarks: any[], exerciseType: string): string[] {
const tips: string[] = [];
const leftShoulder = landmarks[11];
const rightShoulder = landmarks[12];
if (Math.abs(leftShoulder.y - rightShoulder.y) > 0.05) {
tips.push("⚠️ 双肩不平衡,注意保持肩膀水平");
}
const leftHip = landmarks[23];
const rightHip = landmarks[24];
if (Math.abs(leftHip.y - rightHip.y) > 0.05) {
tips.push("⚠️ 重心不稳,注意保持髋部水平");
}
const nose = landmarks[0];
const midShoulder = { x: (leftShoulder.x + rightShoulder.x) / 2, y: (leftShoulder.y + rightShoulder.y) / 2 };
if (Math.abs(nose.x - midShoulder.x) > 0.08) {
tips.push("⚠️ 头部偏移,保持头部在身体中心线上");
}
if (exerciseType === "forehand") {
const rightElbow = landmarks[14];
const rightWrist = landmarks[16];
if (rightElbow.y > rightShoulder.y + 0.15) {
tips.push("💡 正手:抬高肘部,保持手臂在肩膀高度");
}
}
if (exerciseType === "serve") {
const rightWrist = landmarks[16];
if (rightWrist.y > rightShoulder.y) {
tips.push("💡 发球:手臂需要充分伸展向上");
}
}
if (tips.length === 0) {
tips.push("✅ 姿势良好,继续保持!");
}
return tips;
}