import { useAuth } from "@/_core/hooks/useAuth"; import { trpc } from "@/lib/trpc"; import { createMediaMarker, createMediaSession, finalizeMediaSession, formatRecordingTime, getMediaSession, pickBitrate, pickRecorderMimeType, signalMediaSession, type MediaMarker, type MediaSession, uploadMediaSegment, } from "@/lib/media"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Input } from "@/components/ui/input"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Progress } from "@/components/ui/progress"; import { Slider } from "@/components/ui/slider"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { useBackgroundTask } from "@/hooks/useBackgroundTask"; import { toast } from "sonner"; import { ACTION_LABELS as RECOGNIZED_ACTION_LABELS, type ActionObservation, type ActionType, type TrackingState, recognizeActionFrame, stabilizeActionFrame, } from "@/lib/actionRecognition"; import { applyTrackZoom, getCameraVideoConstraints, readTrackZoomState } from "@/lib/camera"; import { formatDateTimeShanghai } from "@/lib/time"; import { Activity, Camera, Circle, CloudUpload, Download, FlipHorizontal, Loader2, Maximize2, Minus, Minimize2, MonitorUp, Plus, Scissors, ShieldAlert, Smartphone, Sparkles, ListTodo, Video, VideoOff, Wifi, WifiOff, Zap, } from "lucide-react"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; type RecorderMode = | "idle" | "preparing" | "recording" | "reconnecting" | "finalizing" | "archived"; type PendingSegment = { sequence: number; durationMs: number; blob: Blob; }; const SEGMENT_LENGTH_MS = 60_000; const MOTION_SAMPLE_MS = 1_500; const MOTION_THRESHOLD = 18; const MOTION_COOLDOWN_MS = 8_000; const ACTION_SAMPLE_MS = 2_500; const INVALID_RECORDING_WINDOW_MS = 60_000; const QUALITY_PRESETS = { economy: { label: "节省流量", subtitle: "1.0 Mbps · 适合移动网络", description: "优先降低上传流量,保留基础训练画面", }, balanced: { label: "均衡模式", subtitle: "1.4-1.9 Mbps · 推荐", description: "Chrome 录制与推流的默认平衡配置", }, clarity: { label: "清晰优先", subtitle: "2.5 Mbps · Wi-Fi 推荐", description: "保留更多动作细节,适合回放和分析", }, } as const; function waitForIceGathering(peer: RTCPeerConnection) { if (peer.iceGatheringState === "complete") { return Promise.resolve(); } return new Promise((resolve) => { const handleStateChange = () => { if (peer.iceGatheringState === "complete") { peer.removeEventListener("icegatheringstatechange", handleStateChange); resolve(); } }; peer.addEventListener("icegatheringstatechange", handleStateChange); }); } function isMobileDevice() { return ( /Android|iPhone|iPad|iPod/i.test(navigator.userAgent) || window.matchMedia("(max-width: 768px)").matches ); } function getArchiveProgress(session: MediaSession | null) { if (!session) return 0; switch (session.archiveStatus) { case "queued": return 45; case "processing": return 78; case "completed": return 100; case "failed": return 100; default: return 20; } } function getArchivePhaseLabel(mode: RecorderMode, session: MediaSession | null, taskProgress?: number | null) { if (mode === "finalizing" && !taskProgress) { return "正在提交归档任务"; } if (session?.archiveStatus === "completed") { return "回放已生成"; } if (session?.archiveStatus === "failed") { return "归档失败"; } if (session?.archiveStatus === "processing") { return "正在生成回放"; } if (session?.archiveStatus === "queued") { return "正在合并分段"; } return "等待归档"; } function formatFileSize(bytes: number) { if (!bytes) return "0 MB"; return `${(bytes / 1024 / 1024).toFixed(bytes > 20 * 1024 * 1024 ? 1 : 2)} MB`; } function createActionSummary(): Record { return { forehand: 0, backhand: 0, serve: 0, volley: 0, overhead: 0, slice: 0, lob: 0, unknown: 0, }; } function summarizeActions(actionSummary: Record) { return Object.entries(actionSummary) .filter(([actionType, count]) => actionType !== "unknown" && count > 0) .sort((left, right) => right[1] - left[1]) .map(([actionType, count]) => ({ actionType: actionType as ActionType, label: RECOGNIZED_ACTION_LABELS[actionType as ActionType] || actionType, count, })); } export default function Recorder() { const { user } = useAuth(); const utils = trpc.useUtils(); const runtimeQuery = trpc.analysis.runtimeGet.useQuery(undefined, { refetchInterval: 1000, refetchIntervalInBackground: true, }); const finalizeTaskMutation = trpc.task.createMediaFinalize.useMutation({ onSuccess: (data) => { setArchiveTaskId(data.taskId); toast.success("录制归档任务已提交"); }, onError: (error) => { toast.error(`录制归档任务提交失败: ${error.message}`); }, }); const liveVideoRef = useRef(null); const playbackVideoRef = useRef(null); const motionCanvasRef = useRef(null); const streamRef = useRef(null); const recorderRef = useRef(null); const peerRef = useRef(null); const actionPoseRef = useRef(null); const currentSegmentStartedAtRef = useRef(0); const recordingStartedAtRef = useRef(0); const segmentSequenceRef = useRef(0); const motionFrameRef = useRef(null); const lastMotionMarkerAtRef = useRef(0); const actionTickerRef = useRef | null>(null); const actionFrameInFlightRef = useRef(false); const actionTrackingRef = useRef({}); const actionHistoryRef = useRef([]); const actionSummaryRef = useRef>(createActionSummary()); const lastRecognizedActionAtRef = useRef(0); const lastActionMarkerAtRef = useRef(0); const latestRecognizedActionRef = useRef("unknown"); const validityOverrideRef = useRef<"valid" | "invalid" | null>(null); const invalidAutoMarkedRef = useRef(false); const pendingUploadsRef = useRef([]); const uploadInFlightRef = useRef(false); const currentSessionRef = useRef(null); const segmentTickerRef = useRef | null>(null); const timerTickerRef = useRef | null>(null); const motionTickerRef = useRef | null>(null); const reconnectTimeoutRef = useRef | null>(null); const modeRef = useRef("idle"); const reconnectAttemptsRef = useRef(0); const facingModeRef = useRef<"user" | "environment">("environment"); const suppressTrackEndedRef = useRef(false); const zoomTargetRef = useRef(1); const [mode, setMode] = useState("idle"); const [qualityPreset, setQualityPreset] = useState("economy"); const [facingMode, setFacingMode] = useState<"user" | "environment">("environment"); const [cameraActive, setCameraActive] = useState(false); const [hasMultipleCameras, setHasMultipleCameras] = useState(false); const [durationMs, setDurationMs] = useState(0); const [sessionMode, setSessionMode] = useState<"practice" | "pk">("practice"); const [isOnline, setIsOnline] = useState(() => navigator.onLine); const [reconnectAttempts, setReconnectAttempts] = useState(0); const [queuedSegments, setQueuedSegments] = useState(0); const [queuedBytes, setQueuedBytes] = useState(0); const [uploadedSegments, setUploadedSegments] = useState(0); const [uploadBytes, setUploadBytes] = useState(0); const [cameraError, setCameraError] = useState(""); const [title, setTitle] = useState(() => `训练录制 ${formatDateTimeShanghai(new Date(), { year: undefined, second: undefined })}`); const [mediaSession, setMediaSession] = useState(null); const [markers, setMarkers] = useState([]); const [connectionState, setConnectionState] = useState("new"); const [immersivePreview, setImmersivePreview] = useState(false); const [archiveTaskId, setArchiveTaskId] = useState(null); const [zoomState, setZoomState] = useState(() => readTrackZoomState(null)); const [actionSummary, setActionSummary] = useState>(() => createActionSummary()); const [currentDetectedAction, setCurrentDetectedAction] = useState("unknown"); const [recordingValidity, setRecordingValidity] = useState<"pending" | "valid" | "invalid">("pending"); const [recordingValidityReason, setRecordingValidityReason] = useState(""); const mobile = useMemo(() => isMobileDevice(), []); const mimeType = useMemo(() => pickRecorderMimeType(), []); const runtimeRole = runtimeQuery.data?.role ?? "idle"; const liveAnalysisRuntime = runtimeQuery.data?.runtimeSession; const liveAnalysisOccupied = runtimeRole === "viewer" && liveAnalysisRuntime?.status === "active"; const currentPlaybackUrl = mediaSession?.playback.mp4Url || mediaSession?.playback.webmUrl || ""; const archiveTaskQuery = useBackgroundTask(archiveTaskId); const archiveProgress = archiveTaskQuery.data?.progress ?? getArchiveProgress(mediaSession); const archivePhaseLabel = getArchivePhaseLabel(mode, mediaSession, archiveTaskQuery.data?.progress); const totalUploadBytes = uploadBytes + queuedBytes; const uploadStillDraining = mode === "finalizing" && (queuedBytes > 0 || queuedSegments > 0 || finalizeTaskMutation.isPending); const archiveRunning = archiveTaskQuery.data?.status === "queued" || archiveTaskQuery.data?.status === "running" || mediaSession?.archiveStatus === "queued" || mediaSession?.archiveStatus === "processing"; const canLeaveRecorderPage = !uploadStillDraining && (archiveRunning || mode === "archived"); const recognizedActionItems = useMemo(() => summarizeActions(actionSummary), [actionSummary]); const syncSessionState = useCallback((session: MediaSession | null) => { currentSessionRef.current = session; setMediaSession(session); setMarkers(session?.markers ?? []); setUploadedSegments(session?.uploadedSegments ?? 0); setUploadBytes(session?.uploadedBytes ?? 0); }, []); const syncQueuedUploadState = useCallback(() => { setQueuedSegments(pendingUploadsRef.current.length); setQueuedBytes(pendingUploadsRef.current.reduce((total, item) => total + item.blob.size, 0)); }, []); useEffect(() => { modeRef.current = mode; }, [mode]); useEffect(() => { reconnectAttemptsRef.current = reconnectAttempts; }, [reconnectAttempts]); useEffect(() => { facingModeRef.current = facingMode; }, [facingMode]); useEffect(() => { if (archiveTaskQuery.data?.status === "succeeded") { void (async () => { if (currentSessionRef.current?.id) { const response = await getMediaSession(currentSessionRef.current.id); syncSessionState(response.session); } setMode("archived"); utils.video.list.invalidate(); toast.success("回放文件已归档完成"); setArchiveTaskId(null); })(); } else if (archiveTaskQuery.data?.status === "failed") { toast.error(`录制归档失败: ${archiveTaskQuery.data.error || "未知错误"}`); setMode("idle"); setArchiveTaskId(null); } }, [archiveTaskQuery.data, syncSessionState, utils.video.list]); const stopTickers = useCallback(() => { if (segmentTickerRef.current) clearInterval(segmentTickerRef.current); if (timerTickerRef.current) clearInterval(timerTickerRef.current); if (motionTickerRef.current) clearInterval(motionTickerRef.current); if (actionTickerRef.current) clearInterval(actionTickerRef.current); segmentTickerRef.current = null; timerTickerRef.current = null; motionTickerRef.current = null; actionTickerRef.current = null; }, []); const closePeer = useCallback(() => { if (peerRef.current) { peerRef.current.onconnectionstatechange = null; peerRef.current.close(); peerRef.current = null; } setConnectionState("closed"); }, []); const stopCamera = useCallback(() => { suppressTrackEndedRef.current = true; if (streamRef.current) { streamRef.current.getTracks().forEach((track) => track.stop()); streamRef.current = null; } if (liveVideoRef.current) { liveVideoRef.current.srcObject = null; } setZoomState(readTrackZoomState(null)); setCameraActive(false); }, []); const syncZoomState = useCallback(async (preferredZoom?: number, providedTrack?: MediaStreamTrack | null) => { const track = providedTrack || streamRef.current?.getVideoTracks()[0] || null; if (!track) { zoomTargetRef.current = 1; setZoomState(readTrackZoomState(null)); return; } let nextState = readTrackZoomState(track); if (nextState.supported && preferredZoom != null && Math.abs(preferredZoom - nextState.current) > nextState.step / 2) { try { nextState = await applyTrackZoom(track, preferredZoom); } catch { nextState = readTrackZoomState(track); } } zoomTargetRef.current = nextState.current; setZoomState(nextState); }, []); const updateZoom = useCallback(async (nextZoom: number) => { const track = streamRef.current?.getVideoTracks()[0] || null; if (!track) return; try { const nextState = await applyTrackZoom(track, nextZoom); zoomTargetRef.current = nextState.current; setZoomState(nextState); } catch (error: any) { toast.error(`镜头缩放调整失败: ${error?.message || "当前设备不支持"}`); } }, []); const stepZoom = useCallback((direction: -1 | 1) => { if (!zoomState.supported) return; const nextZoom = Math.max( zoomState.min, Math.min(zoomState.max, zoomState.current + zoomState.step * direction), ); void updateZoom(nextZoom); }, [updateZoom, zoomState]); const startCamera = useCallback(( async ( nextFacingMode = facingMode, preferredZoom = zoomTargetRef.current, preset: keyof typeof QUALITY_PRESETS = qualityPreset, ) => { if (liveAnalysisOccupied) { const title = liveAnalysisRuntime?.title || "其他设备正在实时分析"; toast.error(`${title},当前设备不能再开启录制摄像头`); throw new Error("当前账号已有其他设备正在实时分析"); } try { if (streamRef.current) { streamRef.current.getTracks().forEach((track) => track.stop()); streamRef.current = null; } const stream = await navigator.mediaDevices.getUserMedia({ video: getCameraVideoConstraints(nextFacingMode, mobile, preset), audio: { echoCancellation: true, noiseSuppression: true, }, }); stream.getVideoTracks().forEach((track) => { track.onended = () => { if (!suppressTrackEndedRef.current && (modeRef.current === "recording" || modeRef.current === "reconnecting")) { void attemptReconnect(); } }; }); suppressTrackEndedRef.current = false; streamRef.current = stream; if (liveVideoRef.current) { liveVideoRef.current.srcObject = stream; await liveVideoRef.current.play(); } await syncZoomState(preferredZoom, stream.getVideoTracks()[0] || null); setCameraError(""); setCameraActive(true); return stream; } catch (error: any) { const message = error?.message || "无法访问摄像头"; setCameraError(message); toast.error(`摄像头启动失败: ${message}`); throw error; } }), [facingMode, liveAnalysisOccupied, liveAnalysisRuntime?.title, mobile, qualityPreset, syncZoomState]); const ensurePreviewStream = useCallback(async () => { if (streamRef.current) { if (liveVideoRef.current && liveVideoRef.current.srcObject !== streamRef.current) { liveVideoRef.current.srcObject = streamRef.current; await liveVideoRef.current.play().catch(() => {}); } await syncZoomState(zoomTargetRef.current); setCameraActive(true); return streamRef.current; } return startCamera(facingModeRef.current, zoomTargetRef.current, qualityPreset); }, [startCamera, syncZoomState]); const refreshDevices = useCallback(async () => { try { const devices = await navigator.mediaDevices.enumerateDevices(); setHasMultipleCameras(devices.filter((device) => device.kind === "videoinput").length > 1); } catch { setHasMultipleCameras(false); } }, []); const processUploadQueue = useCallback(async () => { if (uploadInFlightRef.current || pendingUploadsRef.current.length === 0 || !currentSessionRef.current?.id || !navigator.onLine) { syncQueuedUploadState(); return; } uploadInFlightRef.current = true; try { while (pendingUploadsRef.current.length > 0 && currentSessionRef.current?.id && navigator.onLine) { const nextSegment = pendingUploadsRef.current[0]; const response = await uploadMediaSegment( currentSessionRef.current.id, nextSegment.sequence, nextSegment.durationMs, nextSegment.blob ); pendingUploadsRef.current.shift(); syncQueuedUploadState(); syncSessionState(response.session); } } catch (error: any) { toast.error(`分段上传失败: ${error?.message || "未知错误"}`); } finally { uploadInFlightRef.current = false; } }, [syncQueuedUploadState, syncSessionState]); const enqueueSegment = useCallback(async (blob: Blob, durationForSegmentMs: number) => { if (!blob.size) return; pendingUploadsRef.current.push({ sequence: segmentSequenceRef.current++, durationMs: Math.max(1, durationForSegmentMs), blob, }); syncQueuedUploadState(); await processUploadQueue(); }, [processUploadQueue, syncQueuedUploadState]); const flushPendingSegments = useCallback(async () => { while (pendingUploadsRef.current.length > 0 || uploadInFlightRef.current) { await processUploadQueue(); await new Promise((resolve) => setTimeout(resolve, 200)); } }, [processUploadQueue]); const stopRecorder = useCallback(async () => { stopTickers(); const recorder = recorderRef.current; recorderRef.current = null; if (!recorder || recorder.state === "inactive") { return; } const stopped = new Promise((resolve) => { recorder.addEventListener("stop", () => resolve(), { once: true }); }); try { recorder.requestData(); } catch { // ignore } recorder.stop(); await stopped; }, [stopTickers]); const createManualMarker = useCallback(async ( type: "manual" | "motion" | "action_detected" | "invalid_auto" | "invalid_manual" | "valid_manual", label: string, confidence?: number, ) => { const sessionId = currentSessionRef.current?.id; if (!sessionId) return; const timestampMs = Math.max(0, Date.now() - recordingStartedAtRef.current); const optimisticMarker: MediaMarker = { id: `${type}-${timestampMs}`, type, label, timestampMs, confidence, createdAt: new Date().toISOString(), }; setMarkers((current) => [...current, optimisticMarker].sort((a, b) => a.timestampMs - b.timestampMs)); try { const response = await createMediaMarker(sessionId, { type, label, timestampMs, confidence, }); syncSessionState(response.session); } catch (error: any) { toast.error(`标记失败: ${error?.message || "未知错误"}`); } }, [syncSessionState]); const setValidityState = useCallback(( nextStatus: "pending" | "valid" | "invalid", reason: string, override: "valid" | "invalid" | null = validityOverrideRef.current, ) => { validityOverrideRef.current = override; setRecordingValidity(nextStatus); setRecordingValidityReason(reason); }, []); const startActionSampling = useCallback(async () => { if (typeof window === "undefined") return; const liveVideo = liveVideoRef.current; if (!liveVideo) return; if (!actionPoseRef.current) { const testFactory = ( window as typeof window & { __TEST_MEDIAPIPE_FACTORY__?: () => Promise<{ Pose: any }>; } ).__TEST_MEDIAPIPE_FACTORY__; const { Pose } = testFactory ? await testFactory() : await import("@mediapipe/pose"); const pose = new Pose({ locateFile: (file: string) => `https://cdn.jsdelivr.net/npm/@mediapipe/pose/${file}`, }); pose.setOptions({ modelComplexity: 0, smoothLandmarks: true, enableSegmentation: false, minDetectionConfidence: 0.5, minTrackingConfidence: 0.5, }); pose.onResults((results: { poseLandmarks?: Array<{ x: number; y: number; visibility?: number }> }) => { if (!results.poseLandmarks) { return; } const analyzed = stabilizeActionFrame( recognizeActionFrame(results.poseLandmarks, actionTrackingRef.current, performance.now()), actionHistoryRef.current, ); setCurrentDetectedAction(analyzed.action); if (analyzed.action === "unknown" || analyzed.confidence < 0.55) { return; } latestRecognizedActionRef.current = analyzed.action; lastRecognizedActionAtRef.current = Date.now(); actionSummaryRef.current = { ...actionSummaryRef.current, [analyzed.action]: (actionSummaryRef.current[analyzed.action] || 0) + 1, }; setActionSummary({ ...actionSummaryRef.current }); if (validityOverrideRef.current !== "invalid") { setValidityState("valid", `已识别到 ${RECOGNIZED_ACTION_LABELS[analyzed.action]}`, validityOverrideRef.current); } if (Date.now() - lastActionMarkerAtRef.current >= 15_000) { lastActionMarkerAtRef.current = Date.now(); void createManualMarker("action_detected", `识别到${RECOGNIZED_ACTION_LABELS[analyzed.action]}`, analyzed.confidence); } }); actionPoseRef.current = pose; } const checkInvalidWindow = () => { const elapsedMs = Date.now() - recordingStartedAtRef.current; if (elapsedMs < INVALID_RECORDING_WINDOW_MS || invalidAutoMarkedRef.current) { return; } if (Date.now() - lastRecognizedActionAtRef.current < INVALID_RECORDING_WINDOW_MS) { return; } invalidAutoMarkedRef.current = true; setValidityState("invalid", "连续 60 秒未识别到有效动作,已自动标记为无效录制", validityOverrideRef.current); void createManualMarker("invalid_auto", "连续60秒未识别到有效动作,自动标记为无效录制"); }; actionTickerRef.current = setInterval(() => { const video = liveVideoRef.current; if (!video || video.readyState < 2 || !actionPoseRef.current || actionFrameInFlightRef.current) { checkInvalidWindow(); return; } actionFrameInFlightRef.current = true; actionPoseRef.current.send({ image: video }) .catch(() => undefined) .finally(() => { actionFrameInFlightRef.current = false; checkInvalidWindow(); }); }, ACTION_SAMPLE_MS); }, [createManualMarker, setValidityState]); const stopActionSampling = useCallback(async () => { if (actionTickerRef.current) { clearInterval(actionTickerRef.current); actionTickerRef.current = null; } if (actionPoseRef.current?.close) { try { await actionPoseRef.current.close(); } catch { // ignore pose teardown failures during recorder stop/reset } } actionPoseRef.current = null; actionFrameInFlightRef.current = false; }, []); const updateRecordingValidity = useCallback(async (next: "valid" | "invalid") => { validityOverrideRef.current = next; if (next === "valid") { setValidityState("valid", "已手工恢复为有效录制", "valid"); invalidAutoMarkedRef.current = false; await createManualMarker("valid_manual", "手工恢复为有效录制"); return; } setValidityState("invalid", "已手工标记为无效录制", "invalid"); await createManualMarker("invalid_manual", "手工标记为无效录制"); }, [createManualMarker, setValidityState]); const sampleMotion = useCallback(() => { const video = liveVideoRef.current; const canvas = motionCanvasRef.current; if (!video || !canvas || video.readyState < 2) { return; } const ctx = canvas.getContext("2d", { willReadFrequently: true }); if (!ctx) { return; } const width = 160; const height = 90; canvas.width = width; canvas.height = height; ctx.drawImage(video, 0, 0, width, height); const frame = ctx.getImageData(0, 0, width, height).data; const lastFrame = motionFrameRef.current; if (lastFrame) { let totalDifference = 0; let samples = 0; for (let index = 0; index < frame.length; index += 16) { totalDifference += Math.abs(frame[index] - lastFrame[index]); samples += 1; } const averageDifference = samples ? totalDifference / samples : 0; const now = Date.now(); if (averageDifference >= MOTION_THRESHOLD && now - lastMotionMarkerAtRef.current >= MOTION_COOLDOWN_MS) { lastMotionMarkerAtRef.current = now; void createManualMarker("motion", "自动关键片段", Math.min(0.98, averageDifference / 100)); } } motionFrameRef.current = new Uint8ClampedArray(frame); }, [createManualMarker]); const startRealtimePush = useCallback(async (stream: MediaStream, sessionId: string) => { closePeer(); const peer = new RTCPeerConnection({ iceServers: [{ urls: "stun:stun.l.google.com:19302" }], }); peerRef.current = peer; setConnectionState("connecting"); stream.getTracks().forEach((track) => { peer.addTrack(track, stream); }); peer.onconnectionstatechange = () => { setConnectionState(peer.connectionState); if (peer.connectionState === "disconnected" && (modeRef.current === "recording" || modeRef.current === "reconnecting")) { void attemptReconnect(); } }; const offer = await peer.createOffer(); await peer.setLocalDescription(offer); await waitForIceGathering(peer); const answer = await signalMediaSession(sessionId, { sdp: peer.localDescription?.sdp || "", type: peer.localDescription?.type || "offer", }); await peer.setRemoteDescription({ type: answer.type as RTCSdpType, sdp: answer.sdp, }); }, [closePeer]); const startRecorderLoop = useCallback((stream: MediaStream) => { const recorder = new MediaRecorder(stream, { mimeType, videoBitsPerSecond: pickBitrate(qualityPreset, mobile), }); currentSegmentStartedAtRef.current = Date.now(); recorder.ondataavailable = (event) => { if (!event.data.size) { return; } const now = Date.now(); const segmentDuration = now - currentSegmentStartedAtRef.current; currentSegmentStartedAtRef.current = now; void enqueueSegment(event.data, segmentDuration); }; recorder.onerror = () => { void attemptReconnect(); }; recorder.start(); recorderRef.current = recorder; segmentTickerRef.current = setInterval(() => { if (recorder.state === "recording") { recorder.requestData(); } }, SEGMENT_LENGTH_MS); timerTickerRef.current = setInterval(() => { setDurationMs(Date.now() - recordingStartedAtRef.current); }, 250); motionTickerRef.current = setInterval(() => { sampleMotion(); }, MOTION_SAMPLE_MS); }, [enqueueSegment, mimeType, mobile, qualityPreset, sampleMotion]); const attemptReconnect = useCallback(async () => { if (!currentSessionRef.current || modeRef.current === "finalizing" || modeRef.current === "archived") { return; } if (reconnectTimeoutRef.current) { clearTimeout(reconnectTimeoutRef.current); } setMode("reconnecting"); setReconnectAttempts((count) => { const nextCount = count + 1; reconnectAttemptsRef.current = nextCount; return nextCount; }); try { await stopRecorder(); } catch { // ignore } closePeer(); stopCamera(); const nextDelay = Math.min(1_000 * Math.max(1, reconnectAttemptsRef.current), 8_000); reconnectTimeoutRef.current = setTimeout(async () => { try { const stream = await startCamera(); await startRealtimePush(stream, currentSessionRef.current?.id || ""); startRecorderLoop(stream); setMode("recording"); toast.success("摄像头已重新连接,录制继续"); } catch (error: any) { toast.error(`自动重连失败: ${error?.message || "未知错误"}`); if (modeRef.current === "reconnecting") { void attemptReconnect(); } } }, nextDelay); }, [closePeer, startCamera, startRealtimePush, startRecorderLoop, stopCamera, stopRecorder]); const beginRecording = useCallback(async () => { if (!user) { toast.error("请先登录后再开始录制"); return; } if (liveAnalysisOccupied) { const title = liveAnalysisRuntime?.title || "其他设备正在实时分析"; toast.error(`${title},当前设备不能同时开始录制`); return; } try { setMode("preparing"); setMarkers([]); setDurationMs(0); setUploadedSegments(0); setUploadBytes(0); setQueuedSegments(0); setQueuedBytes(0); setReconnectAttempts(0); setArchiveTaskId(null); segmentSequenceRef.current = 0; motionFrameRef.current = null; pendingUploadsRef.current = []; actionTrackingRef.current = {}; actionHistoryRef.current = []; actionSummaryRef.current = createActionSummary(); setActionSummary(createActionSummary()); setCurrentDetectedAction("unknown"); setRecordingValidity("pending"); setRecordingValidityReason("正在抽样动作帧,持续 60 秒未识别到有效动作将自动标记无效。"); validityOverrideRef.current = null; invalidAutoMarkedRef.current = false; latestRecognizedActionRef.current = "unknown"; lastActionMarkerAtRef.current = 0; const stream = await ensurePreviewStream(); const sessionResponse = await createMediaSession({ userId: String(user.id), title: title.trim() || `训练录制 ${formatDateTimeShanghai(new Date())}`, format: "webm", mimeType, qualityPreset, facingMode, deviceKind: mobile ? "mobile" : "desktop", }); syncSessionState(sessionResponse.session); await startRealtimePush(stream, sessionResponse.session.id); recordingStartedAtRef.current = Date.now(); lastRecognizedActionAtRef.current = recordingStartedAtRef.current; startRecorderLoop(stream); await startActionSampling(); setMode("recording"); toast.success("录制已开始,已同步启动实时推流"); } catch (error: any) { setMode("idle"); toast.error(`启动录制失败: ${error?.message || "未知错误"}`); } }, [ ensurePreviewStream, facingMode, liveAnalysisOccupied, liveAnalysisRuntime?.title, mimeType, mobile, qualityPreset, startActionSampling, startRealtimePush, startRecorderLoop, syncSessionState, title, user, ]); const finishRecording = useCallback(async () => { const session = currentSessionRef.current; if (!session) { return; } try { setMode("finalizing"); await stopActionSampling(); await stopRecorder(); await flushPendingSegments(); closePeer(); stopCamera(); const response = await finalizeMediaSession(session.id, { title: title.trim() || session.title, durationMs: Date.now() - recordingStartedAtRef.current, }); syncSessionState(response.session); await finalizeTaskMutation.mutateAsync({ sessionId: session.id, title: title.trim() || session.title, exerciseType: "recording", sessionMode, durationMinutes: Math.max(1, Math.round((Date.now() - recordingStartedAtRef.current) / 60000)), actionCount: Object.entries(actionSummaryRef.current) .filter(([actionType]) => actionType !== "unknown") .reduce((sum, [, count]) => sum + count, 0), actionSummary: actionSummaryRef.current, dominantAction: latestRecognizedActionRef.current !== "unknown" ? latestRecognizedActionRef.current : undefined, validityStatus: recordingValidity === "invalid" ? validityOverrideRef.current === "invalid" ? "invalid_manual" : "invalid_auto" : recordingValidity === "valid" ? validityOverrideRef.current === "valid" ? "valid_manual" : "valid" : "pending", invalidReason: recordingValidity === "invalid" ? recordingValidityReason : undefined, }); toast.success("录制已提交,后台正在整理回放文件"); } catch (error: any) { toast.error(`结束录制失败: ${error?.message || "未知错误"}`); setMode("recording"); } }, [closePeer, finalizeTaskMutation, flushPendingSegments, recordingValidity, recordingValidityReason, sessionMode, stopActionSampling, stopCamera, stopRecorder, syncSessionState, title]); const resetRecorder = useCallback(async () => { if (reconnectTimeoutRef.current) clearTimeout(reconnectTimeoutRef.current); stopTickers(); await stopRecorder().catch(() => {}); closePeer(); stopCamera(); pendingUploadsRef.current = []; uploadInFlightRef.current = false; motionFrameRef.current = null; await stopActionSampling().catch(() => {}); actionTrackingRef.current = {}; actionHistoryRef.current = []; actionSummaryRef.current = createActionSummary(); currentSessionRef.current = null; setArchiveTaskId(null); setMediaSession(null); setMarkers([]); setActionSummary(createActionSummary()); setCurrentDetectedAction("unknown"); setRecordingValidity("pending"); setRecordingValidityReason(""); setDurationMs(0); setQueuedSegments(0); setQueuedBytes(0); setUploadedSegments(0); setUploadBytes(0); setReconnectAttempts(0); setConnectionState("new"); setCameraError(""); setMode("idle"); }, [closePeer, stopActionSampling, stopCamera, stopRecorder, stopTickers]); const flipCamera = useCallback(async () => { const nextFacingMode = facingMode === "user" ? "environment" : "user"; setFacingMode(nextFacingMode); if (mode === "idle" && cameraActive) { stopCamera(); await startCamera(nextFacingMode, zoomTargetRef.current, qualityPreset); } }, [cameraActive, facingMode, mode, qualityPreset, startCamera, stopCamera]); const handleQualityPresetChange = useCallback(async (nextPreset: keyof typeof QUALITY_PRESETS) => { setQualityPreset(nextPreset); if (cameraActive && mode === "idle") { await startCamera(facingModeRef.current, zoomTargetRef.current, nextPreset); } }, [cameraActive, mode, startCamera]); useEffect(() => { void refreshDevices(); navigator.mediaDevices?.addEventListener?.("devicechange", refreshDevices); const handleOnline = () => { setIsOnline(true); toast.success("网络已恢复,继续上传缓存分段"); void processUploadQueue(); }; const handleOffline = () => { setIsOnline(false); toast.warning("网络已断开,录制分段已留在本地队列"); }; window.addEventListener("online", handleOnline); window.addEventListener("offline", handleOffline); return () => { navigator.mediaDevices?.removeEventListener?.("devicechange", refreshDevices); window.removeEventListener("online", handleOnline); window.removeEventListener("offline", handleOffline); }; }, [processUploadQueue, refreshDevices]); useEffect(() => { if (!currentPlaybackUrl || !playbackVideoRef.current || mode !== "archived") { return; } playbackVideoRef.current.load(); }, [currentPlaybackUrl, immersivePreview, mode]); useEffect(() => { if (mode === "archived") { return; } const liveVideo = liveVideoRef.current; if (!liveVideo || !streamRef.current) { return; } if (liveVideo.srcObject !== streamRef.current) { liveVideo.srcObject = streamRef.current; } liveVideo.play().catch(() => {}); }, [cameraActive, immersivePreview, mode]); useEffect(() => { if (!mobile) { setImmersivePreview(false); return; } if (!immersivePreview) { document.body.style.overflow = ""; return; } document.body.style.overflow = "hidden"; return () => { document.body.style.overflow = ""; }; }, [immersivePreview, mobile]); useEffect(() => { return () => { if (reconnectTimeoutRef.current) clearTimeout(reconnectTimeoutRef.current); stopTickers(); void stopActionSampling(); if (recorderRef.current && recorderRef.current.state !== "inactive") { try { recorderRef.current.stop(); } catch { // ignore } } closePeer(); if (streamRef.current) { streamRef.current.getTracks().forEach((track) => track.stop()); } }; }, [closePeer, stopActionSampling, stopTickers]); const statusBadge = useMemo(() => { if (mode === "finalizing") { return { label: "归档处理中", tone: "default" as const, icon: CloudUpload }; } if (mode === "reconnecting") { return { label: "摄像头重连中", tone: "secondary" as const, icon: ShieldAlert }; } if (mode === "recording") { return { label: "录制中", tone: "destructive" as const, icon: Circle }; } if (mode === "archived") { return { label: "回放已就绪", tone: "default" as const, icon: Video }; } return { label: "待机", tone: "outline" as const, icon: Camera }; }, [mode]); const previewTitle = mode === "archived" ? "归档回放" : "实时取景"; const StatusIcon = statusBadge.icon; const previewVideoClassName = immersivePreview && mobile ? "h-full w-full object-contain" : "h-full w-full object-contain"; const railButtonClassName = "h-16 w-16 flex-col gap-1 rounded-[20px] px-2 text-[11px] leading-tight shadow-lg shadow-black/20"; const renderPrimaryActions = (layout: "toolbar" | "rail" = "toolbar") => { const rail = layout === "rail"; const labelFor = (full: string, short: string) => (rail ? short : full); const buttonClass = (tone: "default" | "outline" | "destructive" | "record" = "default") => { if (!rail) { switch (tone) { case "outline": return "h-12 rounded-2xl px-4"; case "destructive": return "h-12 rounded-2xl px-5"; case "record": return "h-12 rounded-2xl bg-red-600 px-5 hover:bg-red-700"; default: return "h-12 rounded-2xl px-5"; } } switch (tone) { case "outline": return `${railButtonClassName} border-white/15 bg-white/8 text-white hover:bg-white/14`; case "destructive": return `${railButtonClassName} bg-red-600 text-white hover:bg-red-700`; case "record": return `${railButtonClassName} bg-red-600 text-white hover:bg-red-700`; default: return `${railButtonClassName} bg-white text-slate-950 hover:bg-white/90`; } }; const iconClass = rail ? "h-5 w-5" : "mr-2 h-4 w-4"; return ( <> {mode === "idle" && ( <> {!cameraActive ? ( ) : ( <> )} {hasMultipleCameras && ( )} )} {(mode === "recording" || mode === "reconnecting") && ( <> )} {mode === "archived" && ( <> {currentPlaybackUrl && ( )} )} {mode === "finalizing" && ( )} ); }; const renderPreviewMedia = () => ( <> {mode === "archived" && currentPlaybackUrl ? (