From 815f96d4e8218ec8c77417251a968e34fe4e0d51 Mon Sep 17 00:00:00 2001 From: cryptocommuniums-afk Date: Sun, 15 Mar 2026 02:11:34 +0800 Subject: [PATCH] Improve live analysis stability and video clip drafting --- README.md | 14 + client/src/pages/LiveCamera.tsx | 61 +++- client/src/pages/Progress.tsx | 11 +- client/src/pages/Videos.tsx | 493 ++++++++++++++++++++++++++++---- docs/FEATURES.md | 8 +- docs/testing.md | 14 + docs/verified-features.md | 10 +- tests/e2e/app.spec.ts | 10 + tests/e2e/helpers/mockApp.ts | 5 + 9 files changed, 570 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 35ea05d..696eff2 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,19 @@ 实时分析页现在采用“识别 + 录制 + 落库”一体化流程： - 浏览器端基于 MediaPipe Pose 自动识别 `forehand / backhand / serve / volley / overhead / slice / lob / unknown` +- 最近 6 帧动作结果会做时序加权稳定化，降低正手/反手/未知动作间的瞬时抖动 - 连续同类动作会自动合并为片段，最长单段不超过 10 秒 - 停止分析后会自动保存动作区间、评分维度、反馈摘要和可选本地录制视频 - 实时分析结果会自动回写训练记录、日训练聚合、成就进度与 NTRP 评分链路 - 移动端支持竖屏最大化预览，主要操作按钮固定在侧边 +## Video Library And PC Editing + +- 视频库支持直接打开 `PC 轻剪辑工作台` +- 轻剪辑支持播放器预览、手动入点/出点、从当前播放位置快速设点 +- 分析关键时刻会自动生成建议片段；即使视频 metadata 尚未返回，也会按分析帧数估算时间轴 +- 剪辑草稿保存在浏览器本地，可导出 JSON 供后续后台剪辑任务或人工复核使用 + ## Online Recording 在线录制模块采用双链路设计： @@ -124,6 +132,12 @@ set -a && source .env && set +a && pnpm exec drizzle-kit migrate - `docs/media-architecture.md` - `docs/frontend-recording.md` +2026-03-15 已在真实环境执行一次重建与 smoke test： + +- `docker compose up -d --build migrate app app-worker` +- Playwright 复测 `https://te.hao.work/login`、`/checkin`、`/videos`、`/recorder`、`/live-camera`、`/admin` +- 复测后关键链路全部通过，确认线上已切换到最新前端与业务版本 + ## Documentation Index - `docs/FEATURES.md`: 当前功能特性与能力边界 diff --git a/client/src/pages/LiveCamera.tsx b/client/src/pages/LiveCamera.tsx index 5b11300..814e4b6 100644 --- a/client/src/pages/LiveCamera.tsx +++ b/client/src/pages/LiveCamera.tsx @@ -76,6 +76,11 @@ type AnalyzedFrame = { feedback: string[]; }; +type ActionObservation = { + action: ActionType; + confidence: number; +}; + const ACTION_META: Record = { forehand: { label: "正手挥拍", tone: "bg-emerald-500/10 text-emerald-700", accent: "bg-emerald-500" }, backhand: { label: "反手挥拍", tone: "bg-sky-500/10 text-sky-700", accent: "bg-sky-500" }, @@ -184,6 +189,55 @@ function createSegment(action: ActionType, elapsedMs: number, frame: AnalyzedFra }; } +function stabilizeAnalyzedFrame(frame: AnalyzedFrame, history: ActionObservation[]): AnalyzedFrame { + const nextHistory = [...history, { action: frame.action, confidence: frame.confidence }].slice(-6); + history.splice(0, history.length, ...nextHistory); + + const weights = nextHistory.map((_, index) => index + 1); + const actionScores = nextHistory.reduce>((acc, sample, index) => { + const weighted = sample.confidence * weights[index]; + acc[sample.action] = (acc[sample.action] || 0) + weighted; + return acc; + }, { + forehand: 0, + backhand: 0, + serve: 0, + volley: 0, + overhead: 0, + slice: 0, + lob: 0, + unknown: 0, + }); + + const ranked = Object.entries(actionScores).sort((a, b) => b[1] - a[1]) as Array<[ActionType, number]>; + const [winner = "unknown", winnerScore = 0] = ranked[0] || []; + const [, runnerScore = 0] = ranked[1] || []; + const winnerSamples = nextHistory.filter((sample) => sample.action === winner); + const averageConfidence = winnerSamples.length > 0 + ? winnerSamples.reduce((sum, sample) => sum + sample.confidence, 0) / winnerSamples.length + : frame.confidence; + + const stableAction = + winner === "unknown" && frame.action !== "unknown" && frame.confidence >= 0.52 + ? frame.action + : winnerScore - runnerScore < 0.2 && frame.confidence >= 0.65 + ? frame.action + : winner; + + const stableConfidence = stableAction === frame.action + ? Math.max(frame.confidence, averageConfidence) + : averageConfidence; + + return { + ...frame, + action: stableAction, + confidence: clamp(stableConfidence, 0, 1), + feedback: stableAction === "unknown" + ? ["系统正在继续观察，当前窗口内未形成稳定动作特征。", ...frame.feedback].slice(0, 3) + : frame.feedback, + }; +} + function analyzePoseFrame(landmarks: Point[], tracking: TrackingState, timestamp: number): AnalyzedFrame { const nose = landmarks[0]; const leftShoulder = landmarks[11]; @@ -428,6 +482,7 @@ export default function LiveCamera() { const animationRef = useRef(0); const sessionStartedAtRef = useRef(0); const trackingRef = useRef({}); + const actionHistoryRef = useRef([]); const currentSegmentRef = useRef(null); const segmentsRef = useRef([]); const frameSamplesRef = useRef([]); @@ -746,6 +801,7 @@ export default function LiveCamera() { segmentsRef.current = []; currentSegmentRef.current = null; trackingRef.current = {}; + actionHistoryRef.current = []; frameSamplesRef.current = []; sessionStartedAtRef.current = Date.now(); setDurationMs(0); @@ -785,7 +841,10 @@ export default function LiveCamera() { drawOverlay(canvas, results.poseLandmarks); if (!results.poseLandmarks) return; - const analyzed = analyzePoseFrame(results.poseLandmarks, trackingRef.current, performance.now()); + const analyzed = stabilizeAnalyzedFrame( + analyzePoseFrame(results.poseLandmarks, trackingRef.current, performance.now()), + actionHistoryRef.current, + ); const elapsedMs = Date.now() - sessionStartedAtRef.current; appendFrameToSegment(analyzed, elapsedMs); frameSamplesRef.current.push(analyzed.score); diff --git a/client/src/pages/Progress.tsx b/client/src/pages/Progress.tsx index b0c8d5a..f07746e 100644 --- a/client/src/pages/Progress.tsx +++ b/client/src/pages/Progress.tsx @@ -4,7 +4,7 @@ import { Card, CardContent, CardHeader, CardTitle, CardDescription } from "@/com import { Badge } from "@/components/ui/badge"; import { Skeleton } from "@/components/ui/skeleton"; import { Button } from "@/components/ui/button"; -import { Activity, Calendar, CheckCircle2, Clock, TrendingUp, Target } from "lucide-react"; +import { Activity, Calendar, CheckCircle2, Clock, TrendingUp, Target, Sparkles } from "lucide-react"; import { ResponsiveContainer, BarChart, Bar, XAxis, YAxis, CartesianGrid, Tooltip, LineChart, Line, Legend @@ -95,6 +95,14 @@ export default function Progress() {

{analyses?.length || 0}次

+ + +
+ 实时分析 +
+

{stats?.recentLiveSessions?.length || 0}条

+ +
@@ -183,6 +191,7 @@ export default function Progress() {

{new Date(record.trainingDate || record.createdAt).toLocaleDateString("zh-CN")} {record.durationMinutes ? ` · ${record.durationMinutes}分钟` : ""} + {record.sourceType ? ` · ${record.sourceType}` : ""}

diff --git a/client/src/pages/Videos.tsx b/client/src/pages/Videos.tsx index 3b7e22d..7b63a54 100644 --- a/client/src/pages/Videos.tsx +++ b/client/src/pages/Videos.tsx @@ -1,12 +1,39 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useAuth } from "@/_core/hooks/useAuth"; import { trpc } from "@/lib/trpc"; -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; import { Skeleton } from "@/components/ui/skeleton"; -import { Video, Play, BarChart3, Clock, Zap, ChevronRight, FileVideo } from "lucide-react"; +import { Slider } from "@/components/ui/slider"; +import { Textarea } from "@/components/ui/textarea"; +import { Input } from "@/components/ui/input"; +import { toast } from "sonner"; +import { + BarChart3, + Clock, + Download, + FileVideo, + Play, + PlayCircle, + Scissors, + Sparkles, + Trash2, + Video, + Zap, +} from "lucide-react"; import { useLocation } from "wouter"; +type ClipDraft = { + id: string; + startSec: number; + endSec: number; + label: string; + notes: string; + source: "manual" | "suggested"; +}; + const statusMap: Record = { pending: { label: "待分析", color: "bg-yellow-100 text-yellow-700" }, analyzing: { label: "分析中", color: "bg-blue-100 text-blue-700" }, @@ -22,48 +49,192 @@ const exerciseTypeMap: Record = { footwork: "脚步移动", shadow: "影子挥拍", wall: "墙壁练习", + recording: "录制归档", + live_analysis: "实时分析", }; +function formatSeconds(totalSeconds: number) { + const seconds = Math.max(0, Math.floor(totalSeconds)); + const minutes = Math.floor(seconds / 60); + const rest = seconds % 60; + return `${minutes.toString().padStart(2, "0")}:${rest.toString().padStart(2, "0")}`; +} + +function clamp(value: number, min: number, max: number) { + return Math.max(min, Math.min(max, value)); +} + +function localStorageKey(videoId: number) { + return `clip-plan:${videoId}`; +} + +function resolveTimelineDurationSec(analysis: any, durationSec: number) { + if (durationSec > 0) return durationSec; + if (typeof analysis?.durationSec === "number" && analysis.durationSec > 0) return analysis.durationSec; + if (typeof analysis?.durationMs === "number" && analysis.durationMs > 0) return analysis.durationMs / 1000; + if (typeof analysis?.framesAnalyzed === "number" && analysis.framesAnalyzed > 0) { + return Math.max(5, Math.round((analysis.framesAnalyzed / 30) * 10) / 10); + } + return 0; +} + +function buildSuggestedClips(analysis: any, durationSec: number) { + const timelineDurationSec = resolveTimelineDurationSec(analysis, durationSec); + if (!analysis?.keyMoments || !Array.isArray(analysis.keyMoments) || timelineDurationSec <= 0) { + return [] as ClipDraft[]; + } + + const framesAnalyzed = Math.max(analysis.framesAnalyzed || 0, 1); + return analysis.keyMoments.slice(0, 6).map((moment: any, index: number) => { + const centerSec = clamp(((moment.frame || 0) / framesAnalyzed) * timelineDurationSec, 0, timelineDurationSec); + const startSec = clamp(centerSec - 1.5, 0, Math.max(0, timelineDurationSec - 0.5)); + const endSec = clamp(centerSec + 2.5, startSec + 0.5, timelineDurationSec); + return { + id: `suggested-${index}-${moment.frame || index}`, + startSec, + endSec, + label: moment.description || `建议片段 ${index + 1}`, + notes: moment.type ? `来源于分析事件：${moment.type}` : "来源于分析关键时刻", + source: "suggested" as const, + }; + }); +} + +function downloadJson(filename: string, data: unknown) { + const blob = new Blob([JSON.stringify(data, null, 2)], { type: "application/json" }); + const url = URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = url; + link.download = filename; + link.click(); + URL.revokeObjectURL(url); +} + export default function Videos() { - const { user } = useAuth(); + useAuth(); const { data: videos, isLoading } = trpc.video.list.useQuery(); const { data: analyses } = trpc.analysis.list.useQuery(); const [, setLocation] = useLocation(); - const getAnalysis = (videoId: number) => { - return analyses?.find((a: any) => a.videoId === videoId); - }; + const previewRef = useRef(null); + const [editorOpen, setEditorOpen] = useState(false); + const [selectedVideo, setSelectedVideo] = useState(null); + const [videoDurationSec, setVideoDurationSec] = useState(0); + const [playbackSec, setPlaybackSec] = useState(0); + const [clipRange, setClipRange] = useState<[number, number]>([0, 5]); + const [clipLabel, setClipLabel] = useState(""); + const [clipNotes, setClipNotes] = useState(""); + const [clipDrafts, setClipDrafts] = useState([]); + + const getAnalysis = useCallback((videoId: number) => { + return analyses?.find((analysis: any) => analysis.videoId === videoId); + }, [analyses]); + + const activeAnalysis = selectedVideo ? getAnalysis(selectedVideo.id) : null; + const timelineDurationSec = useMemo( + () => resolveTimelineDurationSec(activeAnalysis, videoDurationSec), + [activeAnalysis, videoDurationSec], + ); + const suggestedClips = useMemo( + () => buildSuggestedClips(activeAnalysis, timelineDurationSec), + [activeAnalysis, timelineDurationSec], + ); + + useEffect(() => { + if (!editorOpen || timelineDurationSec <= 0) return; + setClipRange((current) => { + const start = clamp(current[0] ?? 0, 0, Math.max(0, timelineDurationSec - 0.5)); + const minEnd = clamp(start + 0.5, 0.5, timelineDurationSec); + const end = clamp(current[1] ?? Math.min(timelineDurationSec, 5), minEnd, timelineDurationSec); + if (start === current[0] && end === current[1]) { + return current; + } + return [start, end]; + }); + }, [editorOpen, timelineDurationSec]); + + useEffect(() => { + if (!selectedVideo) return; + try { + const saved = localStorage.getItem(localStorageKey(selectedVideo.id)); + if (saved) { + const parsed = JSON.parse(saved) as ClipDraft[]; + setClipDrafts(parsed); + return; + } + } catch { + // Ignore corrupted local clip drafts and fall back to suggested clips. + } + setClipDrafts(suggestedClips); + }, [selectedVideo, suggestedClips]); + + useEffect(() => { + if (!selectedVideo) return; + localStorage.setItem(localStorageKey(selectedVideo.id), JSON.stringify(clipDrafts)); + }, [clipDrafts, selectedVideo]); + + const openEditor = useCallback((video: any) => { + setSelectedVideo(video); + setEditorOpen(true); + setVideoDurationSec(0); + setPlaybackSec(0); + setClipLabel(""); + setClipNotes(""); + setClipRange([0, 5]); + }, []); + + const addClip = useCallback((source: "manual" | "suggested", preset?: ClipDraft) => { + const nextStart = preset?.startSec ?? clipRange[0]; + const nextEnd = preset?.endSec ?? clipRange[1]; + const clip: ClipDraft = { + id: `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, + startSec: nextStart, + endSec: nextEnd, + label: preset?.label || clipLabel || `片段 ${clipDrafts.length + 1}`, + notes: preset?.notes || clipNotes, + source, + }; + + setClipDrafts((current) => [...current, clip].sort((a, b) => a.startSec - b.startSec)); + setClipLabel(""); + setClipNotes(""); + toast.success("片段已加入轻剪辑草稿"); + }, [clipDrafts.length, clipLabel, clipNotes, clipRange]); if (isLoading) { return (
- {[1, 2, 3].map(i => )} + {[1, 2, 3].map((index) => )}
); } return (
-
-
-

训练视频库

-

- 管理您的所有训练视频及分析结果 · 共 {videos?.length || 0} 个视频 -

+
+
+
+

训练视频库

+

+ 集中管理录制归档、上传分析和实时分析视频。桌面端已提供轻剪辑工作台，可按建议片段或手动入点/出点生成剪辑草稿。 +

+
+
+ +
- -
+ {(!videos || videos.length === 0) ? ( - -

还没有训练视频

-

上传您的训练视频，AI将自动分析姿势并给出建议

+ +

还没有训练视频

+

上传训练视频后，这里会自动汇总分析结果，并提供轻剪辑入口。

+ ) : null} + +
- {/* Analysis summary */} - {analysis && ( -
+ {analysis ? ( +
{Math.round(analysis.overallScore || 0)}分
- {(analysis.shotCount ?? 0) > 0 && ( + {(analysis.shotCount ?? 0) > 0 ? (
- {analysis.shotCount}次击球 + {analysis.shotCount} 次击球
- )} - {(analysis.avgSwingSpeed ?? 0) > 0 && ( -
- 速度 {(analysis.avgSwingSpeed ?? 0).toFixed(1)} -
- )} - {(analysis.strokeConsistency ?? 0) > 0 && ( -
+ ) : null} + {(analysis.strokeConsistency ?? 0) > 0 ? ( +
一致性 {Math.round(analysis.strokeConsistency ?? 0)}%
- )} + ) : null} + {Array.isArray(analysis.keyMoments) && analysis.keyMoments.length > 0 ? ( + + + {analysis.keyMoments.length} 个建议片段 + + ) : null}
- )} + ) : null}
@@ -145,6 +326,222 @@ export default function Videos() { })} )} + + + + + + + PC 轻剪辑工作台 + + + 支持手动设置入点/出点、按分析关键时刻生成建议片段，并把剪辑草稿导出为 JSON。 + + + + {selectedVideo ? ( +
+
+
+
+ + + + 片段设置 + 建议先在播放器中定位，再设置入点和出点。 + + +
+
+
当前播放
+
{formatSeconds(playbackSec)}
+
+
+
入点
+
{formatSeconds(clipRange[0])}
+
+
+
出点
+
{formatSeconds(clipRange[1])}
+
+
+ + {timelineDurationSec > 0 ? ( + { + if (value.length === 2) { + setClipRange([value[0] || 0, value[1] || Math.max(0.5, timelineDurationSec)]); + } + }} + /> + ) : null} + +
+ + + +
+ +
+ setClipLabel(event.target.value)} + placeholder="片段名称，例如：正手节奏稳定段" + className="h-11 rounded-2xl" + /> + +
+