feat: async task pipeline for media and llm workflows

这个提交包含在:
cryptocommuniums-afk
2026-03-15 00:12:26 +08:00
父节点 1cc863e60e
当前提交 20e183d2da
修改 36 个文件,包含 1961 行新增339 行删除

查看文件

@@ -7,10 +7,12 @@ import { Input } from "@/components/ui/input";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { Progress } from "@/components/ui/progress";
import { Badge } from "@/components/ui/badge";
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
import { useBackgroundTask } from "@/hooks/useBackgroundTask";
import { toast } from "sonner";
import {
Upload, Video, Loader2, Play, Pause, RotateCcw,
Zap, Target, Activity, TrendingUp, Eye
Zap, Target, Activity, TrendingUp, Eye, ListTodo
} from "lucide-react";
import { Streamdown } from "streamdown";
@@ -39,6 +41,8 @@ export default function Analysis() {
const [analysisProgress, setAnalysisProgress] = useState(0);
const [analysisResult, setAnalysisResult] = useState<AnalysisResult | null>(null);
const [corrections, setCorrections] = useState<string>("");
const [correctionReport, setCorrectionReport] = useState<any>(null);
const [correctionTaskId, setCorrectionTaskId] = useState<string | null>(null);
const [showSkeleton, setShowSkeleton] = useState(false);
const videoRef = useRef<HTMLVideoElement>(null);
const canvasRef = useRef<HTMLCanvasElement>(null);
@@ -55,7 +59,16 @@ export default function Analysis() {
utils.rating.history.invalidate();
},
});
const correctionMutation = trpc.analysis.getCorrections.useMutation();
const correctionMutation = trpc.analysis.getCorrections.useMutation({
onSuccess: (data) => {
setCorrectionTaskId(data.taskId);
toast.success("动作纠正任务已提交");
},
onError: (error) => {
toast.error("动作纠正任务提交失败: " + error.message);
},
});
const correctionTaskQuery = useBackgroundTask(correctionTaskId);
const handleFileSelect = (e: React.ChangeEvent<HTMLInputElement>) => {
const file = e.target.files?.[0];
@@ -73,8 +86,22 @@ export default function Analysis() {
setVideoUrl(URL.createObjectURL(file));
setAnalysisResult(null);
setCorrections("");
setCorrectionReport(null);
setCorrectionTaskId(null);
};
useEffect(() => {
if (correctionTaskQuery.data?.status === "succeeded") {
const result = correctionTaskQuery.data.result as { corrections?: string; report?: any } | null;
setCorrections(result?.corrections || "暂无建议");
setCorrectionReport(result?.report || null);
setCorrectionTaskId(null);
} else if (correctionTaskQuery.data?.status === "failed") {
toast.error(`动作纠正失败: ${correctionTaskQuery.data.error || "未知错误"}`);
setCorrectionTaskId(null);
}
}, [correctionTaskQuery.data]);
const analyzeVideo = useCallback(async () => {
if (!videoRef.current || !canvasRef.current || !videoFile) return;
@@ -267,6 +294,8 @@ export default function Analysis() {
};
setAnalysisResult(result);
setCorrections("");
setCorrectionReport(null);
// Upload video and save analysis
const reader = new FileReader();
@@ -293,13 +322,12 @@ export default function Analysis() {
};
reader.readAsDataURL(videoFile);
// Get AI corrections
const snapshots = await extractFrameSnapshots(videoUrl);
correctionMutation.mutate({
poseMetrics: result.poseMetrics,
exerciseType,
detectedIssues: result.detectedIssues,
}, {
onSuccess: (data) => setCorrections(data.corrections as string),
imageDataUrls: snapshots,
});
pose.close();
@@ -318,6 +346,16 @@ export default function Analysis() {
<p className="text-muted-foreground text-sm mt-1">AI姿势识别与矫正反馈</p>
</div>
{(correctionMutation.isPending || correctionTaskQuery.data?.status === "queued" || correctionTaskQuery.data?.status === "running") ? (
<Alert>
<ListTodo className="h-4 w-4" />
<AlertTitle></AlertTitle>
<AlertDescription>
</AlertDescription>
</Alert>
) : null}
{/* Upload section */}
<Card className="border-0 shadow-sm">
<CardHeader className="pb-3">
@@ -532,7 +570,12 @@ export default function Analysis() {
{correctionMutation.isPending ? (
<div className="flex items-center gap-2 text-muted-foreground">
<Loader2 className="h-4 w-4 animate-spin" />
<span className="text-sm">AI正在生成矫正建议...</span>
<span className="text-sm">...</span>
</div>
) : correctionTaskQuery.data?.status === "queued" || correctionTaskQuery.data?.status === "running" ? (
<div className="flex items-center gap-2 text-muted-foreground">
<Loader2 className="h-4 w-4 animate-spin" />
<span className="text-sm">{correctionTaskQuery.data.message || "AI正在后台生成多模态矫正建议..."}</span>
</div>
) : corrections ? (
<div className="prose prose-sm max-w-none">
@@ -543,6 +586,24 @@ export default function Analysis() {
)}
</CardContent>
</Card>
{correctionReport?.priorityFixes?.length ? (
<Card className="border-0 shadow-sm">
<CardHeader className="pb-2">
<CardTitle className="text-base"></CardTitle>
</CardHeader>
<CardContent className="space-y-3">
{correctionReport.priorityFixes.map((item: any, index: number) => (
<div key={`${item.title}-${index}`} className="rounded-xl border p-3">
<p className="font-medium text-sm">{item.title}</p>
<p className="mt-1 text-sm text-muted-foreground">{item.why}</p>
<p className="mt-2 text-sm"><strong></strong>{item.howToPractice}</p>
<p className="mt-1 text-xs text-primary"><strong></strong>{item.successMetric}</p>
</div>
))}
</CardContent>
</Card>
) : null}
</>
)}
</div>
@@ -667,3 +728,39 @@ function averageAngles(anglesHistory: any[]) {
}
return avg;
}
async function extractFrameSnapshots(sourceUrl: string) {
if (!sourceUrl) return [];
const video = document.createElement("video");
video.src = sourceUrl;
video.muted = true;
video.playsInline = true;
video.crossOrigin = "anonymous";
await new Promise<void>((resolve, reject) => {
video.onloadedmetadata = () => resolve();
video.onerror = () => reject(new Error("无法读取视频元数据"));
});
const canvas = document.createElement("canvas");
canvas.width = video.videoWidth || 1280;
canvas.height = video.videoHeight || 720;
const ctx = canvas.getContext("2d");
if (!ctx) return [];
const duration = Math.max(video.duration || 0, 1);
const checkpoints = [0.15, 0.5, 0.85].map((ratio) => Math.min(duration - 0.05, duration * ratio)).filter((time, index, array) => time >= 0 && array.indexOf(time) === index);
const snapshots: string[] = [];
for (const checkpoint of checkpoints) {
await new Promise<void>((resolve) => {
video.onseeked = () => resolve();
video.currentTime = checkpoint;
});
ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
snapshots.push(canvas.toDataURL("image/jpeg", 0.82));
}
return snapshots;
}

查看文件

@@ -51,11 +51,10 @@ export default function Dashboard() {
return (
<div className="space-y-6">
{/* Welcome header */}
<div className="flex flex-col sm:flex-row sm:items-center sm:justify-between gap-4">
<div>
<h1 className="text-2xl font-bold tracking-tight" data-testid="dashboard-title">
{user?.name || "球友"}
{user?.name || "未命名用户"}
</h1>
<div className="flex items-center gap-3 mt-2">
<NTRPBadge rating={stats?.ntrpRating || 1.5} />

查看文件

@@ -39,7 +39,7 @@ export default function Login() {
try {
const data = await loginMutation.mutateAsync({ username: username.trim() });
const user = await syncAuthenticatedUser(data.user);
toast.success(data.isNew ? `欢迎加入,${user.name}` : `欢迎回来,${user.name}`);
toast.success(data.isNew ? `已创建用户:${user.name}` : `已登录:${user.name}`);
setLocation("/dashboard");
} catch (err) {
const message = err instanceof Error ? err.message : "未知错误";

查看文件

@@ -18,6 +18,8 @@ import { Button } from "@/components/ui/button";
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
import { Input } from "@/components/ui/input";
import { Progress } from "@/components/ui/progress";
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
import { useBackgroundTask } from "@/hooks/useBackgroundTask";
import { toast } from "sonner";
import {
Activity,
@@ -34,6 +36,7 @@ import {
ShieldAlert,
Smartphone,
Sparkles,
ListTodo,
Video,
VideoOff,
Wifi,
@@ -126,7 +129,16 @@ function formatFileSize(bytes: number) {
export default function Recorder() {
const { user } = useAuth();
const registerExternalMutation = trpc.video.registerExternal.useMutation();
const utils = trpc.useUtils();
const finalizeTaskMutation = trpc.task.createMediaFinalize.useMutation({
onSuccess: (data) => {
setArchiveTaskId(data.taskId);
toast.success("录制归档任务已提交");
},
onError: (error) => {
toast.error(`录制归档任务提交失败: ${error.message}`);
},
});
const liveVideoRef = useRef<HTMLVideoElement>(null);
const playbackVideoRef = useRef<HTMLVideoElement>(null);
@@ -142,11 +154,9 @@ export default function Recorder() {
const pendingUploadsRef = useRef<PendingSegment[]>([]);
const uploadInFlightRef = useRef(false);
const currentSessionRef = useRef<MediaSession | null>(null);
const registeredSessionIdRef = useRef<string | null>(null);
const segmentTickerRef = useRef<ReturnType<typeof setInterval> | null>(null);
const timerTickerRef = useRef<ReturnType<typeof setInterval> | null>(null);
const motionTickerRef = useRef<ReturnType<typeof setInterval> | null>(null);
const pollTickerRef = useRef<ReturnType<typeof setInterval> | null>(null);
const reconnectTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const modeRef = useRef<RecorderMode>("idle");
const reconnectAttemptsRef = useRef(0);
@@ -170,11 +180,13 @@ export default function Recorder() {
const [markers, setMarkers] = useState<MediaMarker[]>([]);
const [connectionState, setConnectionState] = useState<RTCPeerConnectionState>("new");
const [immersivePreview, setImmersivePreview] = useState(false);
const [archiveTaskId, setArchiveTaskId] = useState<string | null>(null);
const mobile = useMemo(() => isMobileDevice(), []);
const mimeType = useMemo(() => pickRecorderMimeType(), []);
const currentPlaybackUrl = mediaSession?.playback.mp4Url || mediaSession?.playback.webmUrl || "";
const archiveProgress = getArchiveProgress(mediaSession);
const archiveTaskQuery = useBackgroundTask(archiveTaskId);
const syncSessionState = useCallback((session: MediaSession | null) => {
currentSessionRef.current = session;
@@ -196,6 +208,25 @@ export default function Recorder() {
facingModeRef.current = facingMode;
}, [facingMode]);
useEffect(() => {
if (archiveTaskQuery.data?.status === "succeeded") {
void (async () => {
if (currentSessionRef.current?.id) {
const response = await getMediaSession(currentSessionRef.current.id);
syncSessionState(response.session);
}
setMode("archived");
utils.video.list.invalidate();
toast.success("回放文件已归档完成");
setArchiveTaskId(null);
})();
} else if (archiveTaskQuery.data?.status === "failed") {
toast.error(`录制归档失败: ${archiveTaskQuery.data.error || "未知错误"}`);
setMode("idle");
setArchiveTaskId(null);
}
}, [archiveTaskQuery.data, syncSessionState, utils.video.list]);
const stopTickers = useCallback(() => {
if (segmentTickerRef.current) clearInterval(segmentTickerRef.current);
if (timerTickerRef.current) clearInterval(timerTickerRef.current);
@@ -556,10 +587,10 @@ export default function Recorder() {
setUploadBytes(0);
setQueuedSegments(0);
setReconnectAttempts(0);
setArchiveTaskId(null);
segmentSequenceRef.current = 0;
motionFrameRef.current = null;
pendingUploadsRef.current = [];
registeredSessionIdRef.current = null;
const stream = await ensurePreviewStream();
const sessionResponse = await createMediaSession({
@@ -602,62 +633,19 @@ export default function Recorder() {
durationMs: Date.now() - recordingStartedAtRef.current,
});
syncSessionState(response.session);
toast.success("录制已提交,正在生成回放文件");
await finalizeTaskMutation.mutateAsync({
sessionId: session.id,
title: title.trim() || session.title,
exerciseType: "recording",
});
toast.success("录制已提交,后台正在整理回放文件");
} catch (error: any) {
toast.error(`结束录制失败: ${error?.message || "未知错误"}`);
setMode("recording");
return;
}
if (pollTickerRef.current) {
clearInterval(pollTickerRef.current);
}
pollTickerRef.current = setInterval(async () => {
const current = currentSessionRef.current;
if (!current?.id) {
return;
}
try {
const response = await getMediaSession(current.id);
syncSessionState(response.session);
if (response.session.archiveStatus === "completed") {
if (pollTickerRef.current) clearInterval(pollTickerRef.current);
setMode("archived");
toast.success("回放文件已归档完成");
if (registeredSessionIdRef.current !== response.session.id) {
const playbackUrl = response.session.playback.webmUrl || response.session.playback.mp4Url;
const playbackFormat = response.session.playback.webmUrl ? "webm" : response.session.playback.mp4Url ? "mp4" : "";
if (!playbackUrl || !playbackFormat) {
return;
}
registeredSessionIdRef.current = response.session.id;
await registerExternalMutation.mutateAsync({
title: title.trim() || response.session.title,
url: playbackUrl,
fileKey: `media/sessions/${response.session.id}/recording.${playbackFormat}`,
format: playbackFormat,
fileSize: response.session.playback.webmSize || response.session.playback.mp4Size,
duration: response.session.durationMs / 1000,
exerciseType: "recording",
});
}
}
if (response.session.archiveStatus === "failed") {
if (pollTickerRef.current) clearInterval(pollTickerRef.current);
toast.error(response.session.lastError || "归档失败");
}
} catch {
// keep polling
}
}, 3_000);
}, [closePeer, flushPendingSegments, registerExternalMutation, stopCamera, stopRecorder, syncSessionState, title]);
}, [closePeer, finalizeTaskMutation, flushPendingSegments, stopCamera, stopRecorder, syncSessionState, title]);
const resetRecorder = useCallback(async () => {
if (pollTickerRef.current) clearInterval(pollTickerRef.current);
if (reconnectTimeoutRef.current) clearTimeout(reconnectTimeoutRef.current);
stopTickers();
await stopRecorder().catch(() => {});
@@ -667,7 +655,7 @@ export default function Recorder() {
uploadInFlightRef.current = false;
motionFrameRef.current = null;
currentSessionRef.current = null;
registeredSessionIdRef.current = null;
setArchiveTaskId(null);
setMediaSession(null);
setMarkers([]);
setDurationMs(0);
@@ -755,7 +743,6 @@ export default function Recorder() {
useEffect(() => {
return () => {
if (pollTickerRef.current) clearInterval(pollTickerRef.current);
if (reconnectTimeoutRef.current) clearTimeout(reconnectTimeoutRef.current);
stopTickers();
if (recorderRef.current && recorderRef.current.state !== "inactive") {
@@ -988,6 +975,17 @@ export default function Recorder() {
</div>
</section>
{(finalizeTaskMutation.isPending || archiveTaskQuery.data?.status === "queued" || archiveTaskQuery.data?.status === "running") ? (
<Alert>
<ListTodo className="h-4 w-4" />
<AlertTitle></AlertTitle>
<AlertDescription>
{archiveTaskQuery.data?.message || "录制文件正在后台整理、转码并登记到视频库。"}
</AlertDescription>
</Alert>
) : null}
<div className="grid gap-4 xl:grid-cols-[minmax(0,1.7fr)_minmax(340px,0.9fr)]">
<section className="space-y-4">
<Card className="overflow-hidden border-0 shadow-lg">

查看文件

@@ -1,4 +1,4 @@
import { useState, useMemo } from "react";
import { useEffect, useMemo, useState } from "react";
import { useAuth } from "@/_core/hooks/useAuth";
import { trpc } from "@/lib/trpc";
import { Card, CardContent, CardHeader, CardTitle, CardDescription } from "@/components/ui/card";
@@ -6,10 +6,12 @@ import { Button } from "@/components/ui/button";
import { Badge } from "@/components/ui/badge";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { Skeleton } from "@/components/ui/skeleton";
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
import { useBackgroundTask } from "@/hooks/useBackgroundTask";
import { toast } from "sonner";
import {
Target, Loader2, CheckCircle2, Circle, Clock, Dumbbell,
RefreshCw, Footprints, Hand, ArrowRight, Sparkles
RefreshCw, Footprints, Hand, ArrowRight, Sparkles, ListTodo
} from "lucide-react";
const categoryIcons: Record<string, React.ReactNode> = {
@@ -42,24 +44,26 @@ export default function Training() {
const [skillLevel, setSkillLevel] = useState<"beginner" | "intermediate" | "advanced">("beginner");
const [durationDays, setDurationDays] = useState(7);
const [selectedDay, setSelectedDay] = useState(1);
const [generateTaskId, setGenerateTaskId] = useState<string | null>(null);
const [adjustTaskId, setAdjustTaskId] = useState<string | null>(null);
const utils = trpc.useUtils();
const { data: activePlan, isLoading: planLoading } = trpc.plan.active.useQuery();
const generateTaskQuery = useBackgroundTask(generateTaskId);
const adjustTaskQuery = useBackgroundTask(adjustTaskId);
const generateMutation = trpc.plan.generate.useMutation({
onSuccess: () => {
toast.success("训练计划已生成!");
utils.plan.active.invalidate();
utils.plan.list.invalidate();
onSuccess: (data) => {
setGenerateTaskId(data.taskId);
toast.success("训练计划任务已提交");
},
onError: (err) => toast.error("生成失败: " + err.message),
});
const adjustMutation = trpc.plan.adjust.useMutation({
onSuccess: (data) => {
toast.success("训练计划已调整!");
utils.plan.active.invalidate();
if (data.adjustmentNotes) toast.info("调整说明: " + data.adjustmentNotes);
setAdjustTaskId(data.taskId);
toast.success("训练计划调整任务已提交");
},
onError: (err) => toast.error("调整失败: " + err.message),
});
@@ -81,6 +85,36 @@ export default function Training() {
}, [activePlan, selectedDay]);
const totalDays = activePlan?.durationDays || 7;
const generating = generateMutation.isPending || generateTaskQuery.data?.status === "queued" || generateTaskQuery.data?.status === "running";
const adjusting = adjustMutation.isPending || adjustTaskQuery.data?.status === "queued" || adjustTaskQuery.data?.status === "running";
useEffect(() => {
if (generateTaskQuery.data?.status === "succeeded") {
toast.success("训练计划已生成");
utils.plan.active.invalidate();
utils.plan.list.invalidate();
setGenerateTaskId(null);
} else if (generateTaskQuery.data?.status === "failed") {
toast.error(`训练计划生成失败: ${generateTaskQuery.data.error || "未知错误"}`);
setGenerateTaskId(null);
}
}, [generateTaskQuery.data, utils.plan.active, utils.plan.list]);
useEffect(() => {
if (adjustTaskQuery.data?.status === "succeeded") {
toast.success("训练计划已调整");
utils.plan.active.invalidate();
utils.plan.list.invalidate();
const adjustmentNotes = (adjustTaskQuery.data.result as { adjustmentNotes?: string } | null)?.adjustmentNotes;
if (adjustmentNotes) {
toast.info(`调整说明: ${adjustmentNotes}`);
}
setAdjustTaskId(null);
} else if (adjustTaskQuery.data?.status === "failed") {
toast.error(`训练计划调整失败: ${adjustTaskQuery.data.error || "未知错误"}`);
setAdjustTaskId(null);
}
}, [adjustTaskQuery.data, utils.plan.active, utils.plan.list]);
if (planLoading) {
return (
@@ -100,6 +134,17 @@ export default function Training() {
</div>
</div>
{generating || adjusting ? (
<Alert>
<ListTodo className="h-4 w-4" />
<AlertTitle></AlertTitle>
<AlertDescription>
{generating ? "训练计划正在后台生成。" : "训练计划正在根据最近分析结果调整。"}
</AlertDescription>
</Alert>
) : null}
{!activePlan ? (
/* Generate new plan */
<Card className="border-0 shadow-sm">
@@ -145,11 +190,11 @@ export default function Training() {
<Button
data-testid="training-generate-button"
onClick={() => generateMutation.mutate({ skillLevel, durationDays })}
disabled={generateMutation.isPending}
disabled={generating}
className="w-full sm:w-auto gap-2"
>
{generateMutation.isPending ? (
<><Loader2 className="h-4 w-4 animate-spin" />...</>
{generating ? (
<><Loader2 className="h-4 w-4 animate-spin" />...</>
) : (
<><Sparkles className="h-4 w-4" /></>
)}
@@ -178,10 +223,10 @@ export default function Training() {
variant="outline"
size="sm"
onClick={() => adjustMutation.mutate({ planId: activePlan.id })}
disabled={adjustMutation.isPending}
disabled={adjusting}
className="gap-1"
>
{adjustMutation.isPending ? (
{adjusting ? (
<Loader2 className="h-3 w-3 animate-spin" />
) : (
<RefreshCw className="h-3 w-3" />
@@ -274,11 +319,11 @@ export default function Training() {
onClick={() => {
generateMutation.mutate({ skillLevel, durationDays });
}}
disabled={generateMutation.isPending}
disabled={generating}
className="gap-2"
>
<Sparkles className="h-4 w-4" />
{generating ? <Loader2 className="h-4 w-4 animate-spin" /> : <Sparkles className="h-4 w-4" />}
{generating ? "后台生成中..." : "重新生成计划"}
</Button>
</div>
</>