Add admin vision lab and LLM vision verification

这个提交包含在:
cryptocommuniums-afk
2026-03-15 00:41:09 +08:00
父节点 20e183d2da
当前提交 ad83ce9c68
修改 18 个文件,包含 915 行新增16 行删除

查看文件

@@ -6,6 +6,7 @@ JWT_SECRET=replace-with-strong-secret
VITE_APP_ID=tennis-training-hub
OAUTH_SERVER_URL=
OWNER_OPEN_ID=
ADMIN_USERNAMES=H1
BUILT_IN_FORGE_API_URL=
BUILT_IN_FORGE_API_KEY=
VITE_OAUTH_PORTAL_URL=

查看文件

@@ -40,6 +40,8 @@
- 图片类任务可单独指定 `LLM_VISION_API_URL` / `LLM_VISION_API_KEY` / `LLM_VISION_MODEL`
- 所有图片输入都要求可从公网访问,因此本地相对路径会通过 `APP_PUBLIC_BASE_URL` 规范化为绝对 URL
- 若视觉模型链路不可用,系统会自动回退到结构化指标驱动的文本纠正,避免任务直接失败
- 系统内置“视觉标准图库”页面 `/vision-lab`,可把公网网球参考图入库并保存每次识别结果
- `ADMIN_USERNAMES` 可指定哪些用户名账号拥有 admin 视角,例如 `H1`
## Quick Start
@@ -114,6 +116,7 @@ pnpm exec playwright install chromium
- `DATABASE_URL`
- `JWT_SECRET`
- `ADMIN_USERNAMES`
- `MYSQL_DATABASE`
- `MYSQL_USER`
- `MYSQL_PASSWORD`

查看文件

@@ -19,6 +19,7 @@ import LiveCamera from "./pages/LiveCamera";
import Recorder from "./pages/Recorder";
import Tutorials from "./pages/Tutorials";
import Reminders from "./pages/Reminders";
import VisionLab from "./pages/VisionLab";
function DashboardRoute({ component: Component }: { component: React.ComponentType }) {
return (
@@ -69,6 +70,9 @@ function Router() {
<Route path="/reminders">
<DashboardRoute component={Reminders} />
</Route>
<Route path="/vision-lab">
<DashboardRoute component={VisionLab} />
</Route>
<Route path="/404" component={NotFound} />
<Route component={NotFound} />
</Switch>

查看文件

@@ -23,14 +23,22 @@ import { useIsMobile } from "@/hooks/useMobile";
import {
LayoutDashboard, LogOut, PanelLeft, Target, Video,
Award, Activity, FileVideo, Trophy, Flame, Camera, CircleDot,
BookOpen, Bell
BookOpen, Bell, Microscope
} from "lucide-react";
import { CSSProperties, useEffect, useRef, useState } from "react";
import { useLocation, Redirect } from "wouter";
import { DashboardLayoutSkeleton } from './DashboardLayoutSkeleton';
import { TaskCenter } from "./TaskCenter";
const menuItems = [
type MenuItem = {
icon: typeof LayoutDashboard;
label: string;
path: string;
group: "main" | "analysis" | "stats" | "learn";
adminOnly?: boolean;
};
const menuItems: MenuItem[] = [
{ icon: LayoutDashboard, label: "仪表盘", path: "/dashboard", group: "main" },
{ icon: Target, label: "训练计划", path: "/training", group: "main" },
{ icon: Flame, label: "每日打卡", path: "/checkin", group: "main" },
@@ -43,6 +51,7 @@ const menuItems = [
{ icon: Trophy, label: "排行榜", path: "/leaderboard", group: "stats" },
{ icon: BookOpen, label: "教程库", path: "/tutorials", group: "learn" },
{ icon: Bell, label: "训练提醒", path: "/reminders", group: "learn" },
{ icon: Microscope, label: "视觉测试", path: "/vision-lab", group: "learn", adminOnly: true },
];
const mobileNavItems = [
@@ -111,7 +120,8 @@ function DashboardLayoutContent({
const isCollapsed = state === "collapsed";
const [isResizing, setIsResizing] = useState(false);
const sidebarRef = useRef<HTMLDivElement>(null);
const activeMenuItem = menuItems.find(item => item.path === location);
const visibleMenuItems = menuItems.filter(item => !item.adminOnly || user?.role === "admin");
const activeMenuItem = visibleMenuItems.find(item => item.path === location);
const isMobile = useIsMobile();
useEffect(() => {
@@ -180,7 +190,7 @@ function DashboardLayoutContent({
<SidebarContent className="gap-0">
<SidebarMenu className="px-2 py-1">
{/* Main group */}
{menuItems.filter(i => i.group === "main").map(item => {
{visibleMenuItems.filter(i => i.group === "main").map(item => {
const isActive = location === item.path;
return (
<SidebarMenuItem key={item.path}>
@@ -201,7 +211,7 @@ function DashboardLayoutContent({
{!isCollapsed && <div className="my-2 mx-2 border-t border-border/50" />}
{!isCollapsed && <p className="px-3 text-[10px] font-medium text-muted-foreground uppercase tracking-wider mb-1"></p>}
{menuItems.filter(i => i.group === "analysis").map(item => {
{visibleMenuItems.filter(i => i.group === "analysis").map(item => {
const isActive = location === item.path;
return (
<SidebarMenuItem key={item.path}>
@@ -222,7 +232,7 @@ function DashboardLayoutContent({
{!isCollapsed && <div className="my-2 mx-2 border-t border-border/50" />}
{!isCollapsed && <p className="px-3 text-[10px] font-medium text-muted-foreground uppercase tracking-wider mb-1"></p>}
{menuItems.filter(i => i.group === "stats").map(item => {
{visibleMenuItems.filter(i => i.group === "stats").map(item => {
const isActive = location === item.path;
return (
<SidebarMenuItem key={item.path}>
@@ -243,7 +253,7 @@ function DashboardLayoutContent({
{!isCollapsed && <div className="my-2 mx-2 border-t border-border/50" />}
{!isCollapsed && <p className="px-3 text-[10px] font-medium text-muted-foreground uppercase tracking-wider mb-1"></p>}
{menuItems.filter(i => i.group === "learn").map(item => {
{visibleMenuItems.filter(i => i.group === "learn").map(item => {
const isActive = location === item.path;
return (
<SidebarMenuItem key={item.path}>

查看文件

@@ -0,0 +1,300 @@
import { useEffect, useMemo, useState } from "react";
import { useAuth } from "@/_core/hooks/useAuth";
import { trpc } from "@/lib/trpc";
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
import { Button } from "@/components/ui/button";
import { Badge } from "@/components/ui/badge";
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
import { Skeleton } from "@/components/ui/skeleton";
import { toast } from "sonner";
import { useBackgroundTask } from "@/hooks/useBackgroundTask";
import { Database, Image as ImageIcon, Loader2, Microscope, ShieldCheck, Sparkles } from "lucide-react";
type ReferenceImage = {
id: number;
slug: string;
title: string;
exerciseType: string;
imageUrl: string;
sourcePageUrl: string;
sourceLabel: string;
author: string | null;
license: string | null;
expectedFocus: string[] | null;
tags: string[] | null;
notes: string | null;
};
type VisionRun = {
id: number;
taskId: string;
userId: number;
userName: string | null;
referenceImageId: number | null;
referenceTitle: string | null;
title: string;
exerciseType: string;
imageUrl: string;
status: "queued" | "succeeded" | "failed";
visionStatus: "pending" | "ok" | "fallback" | "failed";
configuredModel: string | null;
expectedFocus: string[] | null;
summary: string | null;
corrections: string | null;
warning: string | null;
error: string | null;
createdAt: Date;
updatedAt: Date;
};
function statusBadge(run: VisionRun) {
if (run.status === "failed" || run.visionStatus === "failed") {
return <Badge variant="destructive"></Badge>;
}
if (run.status === "queued" || run.visionStatus === "pending") {
return <Badge variant="secondary"></Badge>;
}
if (run.visionStatus === "fallback") {
return <Badge variant="outline"></Badge>;
}
return <Badge className="bg-emerald-600 hover:bg-emerald-600"></Badge>;
}
export default function VisionLab() {
const { user } = useAuth();
const utils = trpc.useUtils();
const [activeTaskId, setActiveTaskId] = useState<string | null>(null);
const activeTask = useBackgroundTask(activeTaskId);
const libraryQuery = trpc.vision.library.useQuery();
const runsQuery = trpc.vision.runs.useQuery(
{ limit: 50 },
{ refetchInterval: 4000 }
);
const seedMutation = trpc.vision.seedLibrary.useMutation({
onSuccess: (data) => {
toast.success(`标准图库已就绪,共 ${data.count}`);
utils.vision.library.invalidate();
},
onError: (error) => toast.error(`标准图库初始化失败: ${error.message}`),
});
const runReferenceMutation = trpc.vision.runReference.useMutation({
onSuccess: (data) => {
setActiveTaskId(data.taskId);
toast.success("视觉测试任务已提交");
utils.vision.runs.invalidate();
},
onError: (error) => toast.error(`视觉测试提交失败: ${error.message}`),
});
const runAllMutation = trpc.vision.runAll.useMutation({
onSuccess: (data) => {
toast.success(`已提交 ${data.count} 个视觉测试任务`);
if (data.queued[0]?.taskId) {
setActiveTaskId(data.queued[0].taskId);
}
utils.vision.runs.invalidate();
},
onError: (error) => toast.error(`批量视觉测试提交失败: ${error.message}`),
});
useEffect(() => {
if (activeTask.data?.status === "succeeded" || activeTask.data?.status === "failed") {
utils.vision.runs.invalidate();
setActiveTaskId(null);
}
}, [activeTask.data, utils.vision.runs]);
const references = useMemo(() => (libraryQuery.data ?? []) as ReferenceImage[], [libraryQuery.data]);
const runs = useMemo(() => (runsQuery.data ?? []) as VisionRun[], [runsQuery.data]);
if (libraryQuery.isLoading && runsQuery.isLoading) {
return (
<div className="space-y-6">
<Skeleton className="h-28 w-full" />
<Skeleton className="h-64 w-full" />
<Skeleton className="h-64 w-full" />
</div>
);
}
return (
<div className="space-y-6">
<div className="flex flex-col gap-4 lg:flex-row lg:items-start lg:justify-between">
<div>
<h1 className="text-2xl font-bold tracking-tight"></h1>
<p className="text-sm text-muted-foreground mt-1">
访
</p>
</div>
<div className="flex flex-wrap gap-2">
{user?.role === "admin" ? (
<Button
variant="outline"
onClick={() => seedMutation.mutate()}
disabled={seedMutation.isPending}
className="gap-2"
>
{seedMutation.isPending ? <Loader2 className="h-4 w-4 animate-spin" /> : <Database className="h-4 w-4" />}
</Button>
) : null}
<Button
onClick={() => runAllMutation.mutate()}
disabled={runAllMutation.isPending || references.length === 0}
className="gap-2"
>
{runAllMutation.isPending ? <Loader2 className="h-4 w-4 animate-spin" /> : <Microscope className="h-4 w-4" />}
</Button>
</div>
</div>
{user?.role === "admin" ? (
<Alert>
<ShieldCheck className="h-4 w-4" />
<AlertTitle>Admin </AlertTitle>
<AlertDescription>
`H1` `ADMIN_USERNAMES`
</AlertDescription>
</Alert>
) : (
<Alert>
<ImageIcon className="h-4 w-4" />
<AlertTitle></AlertTitle>
<AlertDescription></AlertDescription>
</Alert>
)}
{activeTask.data?.status === "queued" || activeTask.data?.status === "running" ? (
<Alert>
<Sparkles className="h-4 w-4" />
<AlertTitle></AlertTitle>
<AlertDescription>{activeTask.data.message || "视觉测试正在后台执行。"}</AlertDescription>
</Alert>
) : null}
<section className="space-y-4">
<div className="flex items-center justify-between">
<h2 className="text-lg font-semibold"></h2>
<Badge variant="secondary">{references.length} </Badge>
</div>
<div className="grid gap-4 md:grid-cols-2 xl:grid-cols-3">
{references.map((reference) => (
<Card key={reference.id} className="overflow-hidden border-0 shadow-sm">
<div className="aspect-[4/3] overflow-hidden bg-muted">
<img
src={reference.imageUrl}
alt={reference.title}
className="h-full w-full object-cover"
loading="lazy"
referrerPolicy="no-referrer"
/>
</div>
<CardHeader className="pb-3">
<CardTitle className="text-base">{reference.title}</CardTitle>
<CardDescription className="flex flex-wrap gap-2">
<Badge variant="outline">{reference.exerciseType}</Badge>
{reference.license ? <Badge variant="secondary">{reference.license}</Badge> : null}
</CardDescription>
</CardHeader>
<CardContent className="space-y-3">
{reference.notes ? (
<p className="text-sm text-muted-foreground">{reference.notes}</p>
) : null}
{reference.expectedFocus?.length ? (
<div className="flex flex-wrap gap-2">
{reference.expectedFocus.map((item) => (
<Badge key={item} variant="outline">{item}</Badge>
))}
</div>
) : null}
<div className="flex items-center justify-between gap-3">
<a
href={reference.sourcePageUrl}
target="_blank"
rel="noreferrer"
className="text-sm text-primary underline-offset-4 hover:underline"
>
</a>
<Button
size="sm"
className="gap-2"
onClick={() => runReferenceMutation.mutate({ referenceImageId: reference.id })}
disabled={runReferenceMutation.isPending}
>
{runReferenceMutation.isPending ? <Loader2 className="h-4 w-4 animate-spin" /> : <Microscope className="h-4 w-4" />}
</Button>
</div>
</CardContent>
</Card>
))}
</div>
</section>
<section className="space-y-4">
<div className="flex items-center justify-between">
<h2 className="text-lg font-semibold"></h2>
<Badge variant="secondary">{runs.length} </Badge>
</div>
<div className="grid gap-4">
{runs.map((run) => (
<Card key={run.id} className="border-0 shadow-sm">
<CardContent className="pt-5 space-y-3">
<div className="flex flex-col gap-3 lg:flex-row lg:items-start lg:justify-between">
<div className="space-y-1">
<div className="flex flex-wrap items-center gap-2">
<h3 className="font-semibold">{run.title}</h3>
{statusBadge(run)}
<Badge variant="outline">{run.exerciseType}</Badge>
</div>
<p className="text-xs text-muted-foreground">
{new Date(run.createdAt).toLocaleString("zh-CN")}
{user?.role === "admin" && run.userName ? ` · 提交人:${run.userName}` : ""}
</p>
</div>
{run.configuredModel ? (
<Badge variant="secondary">{run.configuredModel}</Badge>
) : null}
</div>
{run.summary ? <p className="text-sm">{run.summary}</p> : null}
{run.warning ? (
<p className="text-sm text-amber-700">{run.warning}</p>
) : null}
{run.error ? (
<p className="text-sm text-destructive">{run.error}</p>
) : null}
{run.expectedFocus?.length ? (
<div className="flex flex-wrap gap-2">
{run.expectedFocus.map((item) => (
<Badge key={item} variant="outline">{item}</Badge>
))}
</div>
) : null}
{run.corrections ? (
<div className="rounded-xl bg-muted/50 p-3 text-sm leading-6 whitespace-pre-wrap">
{run.corrections}
</div>
) : null}
</CardContent>
</Card>
))}
{runs.length === 0 ? (
<Card className="border-dashed">
<CardContent className="pt-6 text-sm text-muted-foreground">
</CardContent>
</Card>
) : null}
</div>
</section>
</div>
);
}

查看文件

@@ -20,6 +20,7 @@
- 实时摄像头分析:浏览器端调用 MediaPipe,进行姿势识别和反馈展示
- 动作纠正:支持文本纠正和多模态纠正两条链路,统一通过后台任务执行
- 多模态图片输入:上传关键帧后会转换为公网可访问的绝对 URL,再提交给视觉模型
- 视觉标准图库:内置网球公网参考图,可直接发起视觉识别测试并保存结果
- 视频库:集中展示录制结果、上传结果和分析摘要
### 在线录制与媒体链路
@@ -51,6 +52,7 @@
- 统一工作台导航
- 仪表盘、训练、视频、录制、分析等模块一致的布局结构
- 全局任务中心:桌面侧边栏和移动端头部都可查看后台任务
- Admin 视觉测试页:`H1` 这类 admin 用户可查看全部视觉测试数据
- 为后续 PC 粗剪时间线预留媒体域与文档规范
## 架构能力

查看文件

@@ -21,6 +21,7 @@
- Node/tRPC 路由输入校验与权限检查
- LLM 模块请求配置与环境变量回退逻辑
- 视觉模型 per-request model override 能力
- 视觉标准图库路由与 admin/H1 全量可见性逻辑
- 媒体工具函数,例如录制时长格式化与码率选择
### 3. Go 媒体服务测试
@@ -85,6 +86,14 @@ pnpm exec tsx -e 'import "dotenv/config"; import { invokeLLM } from "./server/_c
如果返回模型与 `LLM_VISION_MODEL` 不一致,说明上游网关忽略了视觉模型选择,业务任务会自动回退到文本纠正结果。
视觉标准图库的真实 smoke test 可直接复用内置数据:
- 初始化 `ADMIN_USERNAMES=H1`
- 登录 `H1` 后访问 `/vision-lab`
- 检查标准图是否已经入库
- 运行单张或批量测试,确认结果会写入 `vision_test_runs`
- 若上游视觉网关不可用,记录应显示 `fallback`
## Production smoke checks
部署到宿主机后,建议至少补以下联测:

查看文件

@@ -1,18 +1,19 @@
# Verified Features
本文档记录当前已经通过自动化验证或构建验证的项目。更新时间2026-03-15 00:11 CST。
本文档记录当前已经通过自动化验证或构建验证的项目。更新时间2026-03-15 00:40 CST。
## 最新完整验证记录
- 通过命令:`pnpm verify`
- 验证时间2026-03-15 00:10 CST
- 结果摘要:`pnpm check` 通过,`pnpm test` 通过80/80),`pnpm test:go` 通过,`pnpm build` 通过,`pnpm test:e2e` 通过6/6`pnpm test:llm` 通过
- 验证时间2026-03-15 00:39 CST
- 结果摘要:`pnpm check` 通过,`pnpm test` 通过85/85),`pnpm test:go` 通过,`pnpm build` 通过,`pnpm test:e2e` 通过6/6`pnpm test:llm` 通过
## 生产部署联测
| 项目 | 验证方式 | 状态 |
|------|----------|------|
| `https://te.hao.work/` HTTPS 访问 | `curl -I https://te.hao.work/` | 通过 |
| `https://te.hao.work/vision-lab` 视觉测试页访问 | `curl -I https://te.hao.work/vision-lab` | 通过 |
| `http://te.hao.work:8302/` 4 位端口访问 | `curl -I http://te.hao.work:8302/` | 通过 |
| 站点 TLS 证书 | Let’s Encrypt ECDSA 证书已签发并由宿主机 nginx 加载 | 通过 |
| 生产登录与首次进入工作台 | Playwright 登录真实站点并跳转 `/dashboard` | 通过 |
@@ -79,6 +80,9 @@
| `.env` 中的 `LLM_API_URL` / `LLM_API_KEY` / `LLM_MODEL` | `pnpm test:llm` | 通过 |
| `https://one.hao.work/v1/chat/completions` 联通性 | `pnpm test:llm` 实际返回文本 | 通过 |
| 视觉模型独立配置路径 | `server/_core/llm.test.ts` + 手工 smoke 检查 | 通过 |
| 视觉标准图库入库 | MySQL 中 `vision_reference_images` 已写入 5 张 Commons 网球参考图 | 通过 |
| 视觉测试结果入库 | MySQL 中 `vision_test_runs` 已写入 3 条真实测试结果 | 通过 |
| H1 全量可见性 | `H1` 用户已提升为 `admin`,可读取全部视觉测试记录;Playwright 真实站点检查通过 | 通过 |
## 已知非阻断警告
@@ -86,6 +90,7 @@
- `pnpm build` 仍有 Vite 大 chunk 警告;当前属于性能优化待办,不影响本次产物生成
- Playwright 运行依赖 mocked media/network,不等价于真机摄像头、真实弱网和真实 WebRTC 质量验收
- 当前上游视觉网关可能忽略 `LLM_VISION_MODEL` 并回退为文本模型;服务端已实现自动降级,任务不会因此直接失败
- 2026-03-15 的真实标准图测试中,正手 / 反手 / 发球三条记录均以 `fallback` 完成,说明当前上游视觉网关仍未稳定返回结构化视觉结果
## 当前未纳入自动验证的内容

查看文件

@@ -0,0 +1,43 @@
CREATE TABLE `vision_reference_images` (
`id` int AUTO_INCREMENT NOT NULL,
`slug` varchar(128) NOT NULL,
`title` varchar(256) NOT NULL,
`exerciseType` varchar(64) NOT NULL,
`imageUrl` text NOT NULL,
`sourcePageUrl` text NOT NULL,
`sourceLabel` varchar(128) NOT NULL,
`author` varchar(128),
`license` varchar(128),
`expectedFocus` json,
`tags` json,
`notes` text,
`sortOrder` int NOT NULL DEFAULT 0,
`isPublished` int NOT NULL DEFAULT 1,
`createdAt` timestamp NOT NULL DEFAULT (now()),
`updatedAt` timestamp NOT NULL DEFAULT (now()) ON UPDATE CURRENT_TIMESTAMP,
CONSTRAINT `vision_reference_images_id` PRIMARY KEY(`id`),
CONSTRAINT `vision_reference_images_slug_unique` UNIQUE(`slug`)
);
--> statement-breakpoint
CREATE TABLE `vision_test_runs` (
`id` int AUTO_INCREMENT NOT NULL,
`taskId` varchar(64) NOT NULL,
`userId` int NOT NULL,
`referenceImageId` int,
`title` varchar(256) NOT NULL,
`exerciseType` varchar(64) NOT NULL,
`imageUrl` text NOT NULL,
`status` enum('queued','succeeded','failed') NOT NULL DEFAULT 'queued',
`visionStatus` enum('pending','ok','fallback','failed') NOT NULL DEFAULT 'pending',
`configuredModel` varchar(128),
`expectedFocus` json,
`summary` text,
`corrections` text,
`report` json,
`warning` text,
`error` text,
`createdAt` timestamp NOT NULL DEFAULT (now()),
`updatedAt` timestamp NOT NULL DEFAULT (now()) ON UPDATE CURRENT_TIMESTAMP,
CONSTRAINT `vision_test_runs_id` PRIMARY KEY(`id`),
CONSTRAINT `vision_test_runs_taskId_unique` UNIQUE(`taskId`)
);

查看文件

@@ -43,6 +43,13 @@
"when": 1773504000000,
"tag": "0005_lively_taskmaster",
"breakpoints": true
},
{
"idx": 6,
"version": "5",
"when": 1773510000000,
"tag": "0006_solid_vision_library",
"breakpoints": true
}
]
}

查看文件

@@ -334,3 +334,55 @@ export const backgroundTasks = mysqlTable("background_tasks", {
export type BackgroundTask = typeof backgroundTasks.$inferSelect;
export type InsertBackgroundTask = typeof backgroundTasks.$inferInsert;
/**
* Vision reference library - canonical public tennis images used for multimodal evaluation
*/
export const visionReferenceImages = mysqlTable("vision_reference_images", {
id: int("id").autoincrement().primaryKey(),
slug: varchar("slug", { length: 128 }).notNull().unique(),
title: varchar("title", { length: 256 }).notNull(),
exerciseType: varchar("exerciseType", { length: 64 }).notNull(),
imageUrl: text("imageUrl").notNull(),
sourcePageUrl: text("sourcePageUrl").notNull(),
sourceLabel: varchar("sourceLabel", { length: 128 }).notNull(),
author: varchar("author", { length: 128 }),
license: varchar("license", { length: 128 }),
expectedFocus: json("expectedFocus"),
tags: json("tags"),
notes: text("notes"),
sortOrder: int("sortOrder").default(0).notNull(),
isPublished: int("isPublished").default(1).notNull(),
createdAt: timestamp("createdAt").defaultNow().notNull(),
updatedAt: timestamp("updatedAt").defaultNow().onUpdateNow().notNull(),
});
export type VisionReferenceImage = typeof visionReferenceImages.$inferSelect;
export type InsertVisionReferenceImage = typeof visionReferenceImages.$inferInsert;
/**
* Vision test run history - records each multimodal evaluation against the standard library
*/
export const visionTestRuns = mysqlTable("vision_test_runs", {
id: int("id").autoincrement().primaryKey(),
taskId: varchar("taskId", { length: 64 }).notNull().unique(),
userId: int("userId").notNull(),
referenceImageId: int("referenceImageId"),
title: varchar("title", { length: 256 }).notNull(),
exerciseType: varchar("exerciseType", { length: 64 }).notNull(),
imageUrl: text("imageUrl").notNull(),
status: mysqlEnum("status", ["queued", "succeeded", "failed"]).default("queued").notNull(),
visionStatus: mysqlEnum("visionStatus", ["pending", "ok", "fallback", "failed"]).default("pending").notNull(),
configuredModel: varchar("configuredModel", { length: 128 }),
expectedFocus: json("expectedFocus"),
summary: text("summary"),
corrections: text("corrections"),
report: json("report"),
warning: text("warning"),
error: text("error"),
createdAt: timestamp("createdAt").defaultNow().notNull(),
updatedAt: timestamp("updatedAt").defaultNow().onUpdateNow().notNull(),
});
export type VisionTestRun = typeof visionTestRuns.$inferSelect;
export type InsertVisionTestRun = typeof visionTestRuns.$inferInsert;

查看文件

@@ -9,6 +9,12 @@ const parseBoolean = (value: string | undefined, fallback: boolean) => {
return value === "1" || value.toLowerCase() === "true";
};
const parseList = (value: string | undefined) =>
(value ?? "")
.split(",")
.map((item) => item.trim())
.filter(Boolean);
export const ENV = {
appId: process.env.VITE_APP_ID ?? "",
appPublicBaseUrl: process.env.APP_PUBLIC_BASE_URL ?? "",
@@ -16,6 +22,7 @@ export const ENV = {
databaseUrl: process.env.DATABASE_URL ?? "",
oAuthServerUrl: process.env.OAUTH_SERVER_URL ?? "",
ownerOpenId: process.env.OWNER_OPEN_ID ?? "",
adminUsernames: parseList(process.env.ADMIN_USERNAMES),
isProduction: process.env.NODE_ENV === "production",
forgeApiUrl: process.env.BUILT_IN_FORGE_API_URL ?? "",
forgeApiKey: process.env.BUILT_IN_FORGE_API_KEY ?? "",

查看文件

@@ -9,6 +9,7 @@ import { appRouter } from "../routers";
import { createContext } from "./context";
import { registerMediaProxy } from "./mediaProxy";
import { serveStatic } from "./static";
import { seedTutorials, seedVisionReferenceImages } from "../db";
function isPortAvailable(port: number): Promise<boolean> {
return new Promise(resolve => {
@@ -30,6 +31,9 @@ async function findAvailablePort(startPort: number = 3000): Promise<number> {
}
async function startServer() {
await seedTutorials();
await seedVisionReferenceImages();
const app = express();
const server = createServer(app);
registerMediaProxy(app);

查看文件

@@ -15,6 +15,8 @@ import {
trainingReminders, InsertTrainingReminder,
notificationLog, InsertNotificationLog,
backgroundTasks, InsertBackgroundTask,
visionReferenceImages, InsertVisionReferenceImage,
visionTestRuns, InsertVisionTestRun,
} from "../drizzle/schema";
import { ENV } from './_core/env';
@@ -89,8 +91,9 @@ export async function createUsernameAccount(username: string): Promise<{ user: t
if (existing.length > 0) {
const user = await db.select().from(users).where(eq(users.id, existing[0].userId)).limit(1);
if (user.length > 0) {
await db.update(users).set({ lastSignedIn: new Date() }).where(eq(users.id, user[0].id));
return { user: user[0], isNew: false };
const updatedRole = ENV.adminUsernames.includes(username) ? "admin" : user[0].role;
await db.update(users).set({ lastSignedIn: new Date(), role: updatedRole }).where(eq(users.id, user[0].id));
return { user: { ...user[0], role: updatedRole, lastSignedIn: new Date() }, isNew: false };
}
}
@@ -100,6 +103,7 @@ export async function createUsernameAccount(username: string): Promise<{ user: t
openId,
name: username,
loginMethod: "username",
role: ENV.adminUsernames.includes(username) ? "admin" : "user",
lastSignedIn: new Date(),
ntrpRating: 1.5,
totalSessions: 0,
@@ -443,6 +447,223 @@ export async function getLeaderboard(sortBy: "ntrpRating" | "totalMinutes" | "to
}).from(users).orderBy(desc(sortColumn)).limit(limit);
}
// ===== VISION REFERENCE LIBRARY =====
export const VISION_REFERENCE_SEED_DATA: Omit<
InsertVisionReferenceImage,
"id" | "createdAt" | "updatedAt"
>[] = [
{
slug: "commons-forehand-tennispictures",
title: "标准图:正手挥拍",
exerciseType: "forehand",
imageUrl: "https://commons.wikimedia.org/wiki/Special:FilePath/Forehand.jpg",
sourcePageUrl: "https://commons.wikimedia.org/wiki/File:Forehand.jpg",
sourceLabel: "Wikimedia Commons",
author: "Tennispictures",
license: "CC0 1.0",
expectedFocus: ["引拍完整", "击球臂路径", "肩髋转动", "重心转移"],
tags: ["forehand", "reference", "commons", "stroke"],
notes: "用于检测模型对正手引拍、发力和随挥阶段的描述能力。",
sortOrder: 1,
isPublished: 1,
},
{
slug: "commons-backhand-federer",
title: "标准图:反手挥拍",
exerciseType: "backhand",
imageUrl: "https://commons.wikimedia.org/wiki/Special:FilePath/Backhand_Federer.jpg",
sourcePageUrl: "https://commons.wikimedia.org/wiki/File:Backhand_Federer.jpg",
sourceLabel: "Wikimedia Commons",
author: "Ian Gampon",
license: "CC BY 2.0",
expectedFocus: ["非持拍手收回", "躯干旋转", "拍面路径", "击球点位置"],
tags: ["backhand", "reference", "commons", "stroke"],
notes: "用于检测模型对单反/反手击球阶段和身体协同的判断。",
sortOrder: 2,
isPublished: 1,
},
{
slug: "commons-serena-serve",
title: "标准图:发球",
exerciseType: "serve",
imageUrl: "https://commons.wikimedia.org/wiki/Special:FilePath/Serena_Williams_Serves.JPG",
sourcePageUrl: "https://commons.wikimedia.org/wiki/File:Serena_Williams_Serves.JPG",
sourceLabel: "Wikimedia Commons",
author: "Clavecin",
license: "Public Domain",
expectedFocus: ["抛球与击球点", "肩肘链条", "躯干伸展", "落地重心"],
tags: ["serve", "reference", "commons", "overhead"],
notes: "用于检测模型对发球上举、鞭打和击球点的识别能力。",
sortOrder: 3,
isPublished: 1,
},
{
slug: "commons-volley-lewis",
title: "标准图:网前截击",
exerciseType: "volley",
imageUrl: "https://commons.wikimedia.org/wiki/Special:FilePath/Ernest_w._lewis,_volleying.jpg",
sourcePageUrl: "https://commons.wikimedia.org/wiki/File:Ernest_w._lewis,_volleying.jpg",
sourceLabel: "Wikimedia Commons",
author: "Unknown author",
license: "Public Domain",
expectedFocus: ["拍头稳定", "准备姿态", "身体前压", "短促触球"],
tags: ["volley", "reference", "commons", "net-play"],
notes: "用于检测模型对截击站位和紧凑击球结构的识别能力。",
sortOrder: 4,
isPublished: 1,
},
{
slug: "commons-tiafoe-backhand",
title: "标准图:现代反手参考",
exerciseType: "backhand",
imageUrl: "https://commons.wikimedia.org/wiki/Special:FilePath/Frances_Tiafoe_Backhand.jpg",
sourcePageUrl: "https://commons.wikimedia.org/wiki/File:Frances_Tiafoe_Backhand.jpg",
sourceLabel: "Wikimedia Commons",
author: null,
license: "Wikimedia Commons file license",
expectedFocus: ["双手协同", "脚步支撑", "肩髋分离", "随挥方向"],
tags: ["backhand", "reference", "commons", "modern"],
notes: "补充现代职业选手反手样本,便于比较传统与现代动作语言。",
sortOrder: 5,
isPublished: 1,
},
];
export async function seedVisionReferenceImages() {
const db = await getDb();
if (!db) return;
const existing = await db.select().from(visionReferenceImages).limit(1);
if (existing.length > 0) return;
for (const item of VISION_REFERENCE_SEED_DATA) {
await db.insert(visionReferenceImages).values(item);
}
}
export async function listVisionReferenceImages() {
const db = await getDb();
if (!db) return [];
return db.select().from(visionReferenceImages)
.where(eq(visionReferenceImages.isPublished, 1))
.orderBy(asc(visionReferenceImages.sortOrder), asc(visionReferenceImages.id));
}
export async function getVisionReferenceImageById(id: number) {
const db = await getDb();
if (!db) return undefined;
const result = await db.select().from(visionReferenceImages)
.where(eq(visionReferenceImages.id, id))
.limit(1);
return result[0];
}
export async function createVisionTestRun(run: InsertVisionTestRun) {
const db = await getDb();
if (!db) throw new Error("Database not available");
const result = await db.insert(visionTestRuns).values(run);
return result[0].insertId;
}
export async function listVisionTestRuns(userId?: number, limit = 50) {
const db = await getDb();
if (!db) return [];
const query = db.select({
id: visionTestRuns.id,
taskId: visionTestRuns.taskId,
userId: visionTestRuns.userId,
userName: users.name,
referenceImageId: visionTestRuns.referenceImageId,
referenceTitle: visionReferenceImages.title,
title: visionTestRuns.title,
exerciseType: visionTestRuns.exerciseType,
imageUrl: visionTestRuns.imageUrl,
status: visionTestRuns.status,
visionStatus: visionTestRuns.visionStatus,
configuredModel: visionTestRuns.configuredModel,
expectedFocus: visionTestRuns.expectedFocus,
summary: visionTestRuns.summary,
corrections: visionTestRuns.corrections,
report: visionTestRuns.report,
warning: visionTestRuns.warning,
error: visionTestRuns.error,
createdAt: visionTestRuns.createdAt,
updatedAt: visionTestRuns.updatedAt,
}).from(visionTestRuns)
.leftJoin(users, eq(users.id, visionTestRuns.userId))
.leftJoin(visionReferenceImages, eq(visionReferenceImages.id, visionTestRuns.referenceImageId))
.orderBy(desc(visionTestRuns.createdAt))
.limit(limit);
if (userId == null) {
return query;
}
return db.select({
id: visionTestRuns.id,
taskId: visionTestRuns.taskId,
userId: visionTestRuns.userId,
userName: users.name,
referenceImageId: visionTestRuns.referenceImageId,
referenceTitle: visionReferenceImages.title,
title: visionTestRuns.title,
exerciseType: visionTestRuns.exerciseType,
imageUrl: visionTestRuns.imageUrl,
status: visionTestRuns.status,
visionStatus: visionTestRuns.visionStatus,
configuredModel: visionTestRuns.configuredModel,
expectedFocus: visionTestRuns.expectedFocus,
summary: visionTestRuns.summary,
corrections: visionTestRuns.corrections,
report: visionTestRuns.report,
warning: visionTestRuns.warning,
error: visionTestRuns.error,
createdAt: visionTestRuns.createdAt,
updatedAt: visionTestRuns.updatedAt,
}).from(visionTestRuns)
.leftJoin(users, eq(users.id, visionTestRuns.userId))
.leftJoin(visionReferenceImages, eq(visionReferenceImages.id, visionTestRuns.referenceImageId))
.where(eq(visionTestRuns.userId, userId))
.orderBy(desc(visionTestRuns.createdAt))
.limit(limit);
}
export async function completeVisionTestRun(taskId: string, data: {
visionStatus: "ok" | "fallback";
summary?: string | null;
corrections: string;
report?: unknown;
warning?: string | null;
}) {
const db = await getDb();
if (!db) return;
await db.update(visionTestRuns).set({
status: "succeeded",
visionStatus: data.visionStatus,
summary: data.summary ?? null,
corrections: data.corrections,
report: data.report ?? null,
warning: data.warning ?? null,
error: null,
}).where(eq(visionTestRuns.taskId, taskId));
}
export async function failVisionTestRun(taskId: string, error: string) {
const db = await getDb();
if (!db) return;
await db.update(visionTestRuns).set({
status: "failed",
visionStatus: "failed",
error,
}).where(eq(visionTestRuns.taskId, taskId));
}
// ===== TUTORIAL OPERATIONS =====
export const TUTORIAL_SEED_DATA: Omit<InsertTutorialVideo, "id">[] = [

查看文件

@@ -816,3 +816,80 @@ describe("notification.markAllRead", () => {
await expect(caller.notification.markAllRead()).rejects.toThrow();
});
});
// ===== VISION LIBRARY TESTS =====
describe("vision.library", () => {
it("requires authentication", async () => {
const { ctx } = createMockContext(null);
const caller = appRouter.createCaller(ctx);
await expect(caller.vision.library()).rejects.toThrow();
});
it("returns seeded references for authenticated users", async () => {
const user = createTestUser();
const { ctx } = createMockContext(user);
const caller = appRouter.createCaller(ctx);
const seedSpy = vi.spyOn(db, "seedVisionReferenceImages").mockResolvedValueOnce();
const listSpy = vi.spyOn(db, "listVisionReferenceImages").mockResolvedValueOnce([
{
id: 1,
slug: "ref-1",
title: "标准图:正手挥拍",
exerciseType: "forehand",
imageUrl: "https://example.com/forehand.jpg",
sourcePageUrl: "https://example.com/source",
sourceLabel: "Example",
author: null,
license: null,
expectedFocus: ["肩髋转动"],
tags: ["forehand"],
notes: null,
sortOrder: 1,
isPublished: 1,
createdAt: new Date(),
updatedAt: new Date(),
},
] as any);
const result = await caller.vision.library();
expect(seedSpy).toHaveBeenCalledTimes(1);
expect(listSpy).toHaveBeenCalledTimes(1);
expect(result).toHaveLength(1);
});
});
describe("vision.runs", () => {
it("limits regular users to their own vision test runs", async () => {
const user = createTestUser({ id: 7, role: "user" });
const { ctx } = createMockContext(user);
const caller = appRouter.createCaller(ctx);
const listSpy = vi.spyOn(db, "listVisionTestRuns").mockResolvedValueOnce([]);
await caller.vision.runs({ limit: 20 });
expect(listSpy).toHaveBeenCalledWith(7, 20);
});
it("allows admin users to view all vision test runs", async () => {
const admin = createTestUser({ id: 9, role: "admin", name: "H1" });
const { ctx } = createMockContext(admin);
const caller = appRouter.createCaller(ctx);
const listSpy = vi.spyOn(db, "listVisionTestRuns").mockResolvedValueOnce([]);
await caller.vision.runs({ limit: 30 });
expect(listSpy).toHaveBeenCalledWith(undefined, 30);
});
});
describe("vision.seedLibrary", () => {
it("rejects non-admin users", async () => {
const user = createTestUser({ role: "user" });
const { ctx } = createMockContext(user);
const caller = appRouter.createCaller(ctx);
await expect(caller.vision.seedLibrary()).rejects.toThrow();
});
});

查看文件

@@ -1,9 +1,10 @@
import { COOKIE_NAME, ONE_YEAR_MS } from "@shared/const";
import { getSessionCookieOptions } from "./_core/cookies";
import { systemRouter } from "./_core/systemRouter";
import { publicProcedure, protectedProcedure, router } from "./_core/trpc";
import { adminProcedure, publicProcedure, protectedProcedure, router } from "./_core/trpc";
import { z } from "zod";
import { sdk } from "./_core/sdk";
import { ENV } from "./_core/env";
import { storagePut } from "./storage";
import * as db from "./db";
import { nanoid } from "nanoid";
@@ -271,6 +272,111 @@ export const appRouter = router({
}),
}),
vision: router({
library: protectedProcedure.query(async () => {
await db.seedVisionReferenceImages();
return db.listVisionReferenceImages();
}),
seedLibrary: adminProcedure.mutation(async () => {
await db.seedVisionReferenceImages();
const images = await db.listVisionReferenceImages();
return { count: images.length };
}),
runs: protectedProcedure
.input(z.object({ limit: z.number().min(1).max(100).default(50) }).optional())
.query(async ({ ctx, input }) => {
const limit = input?.limit ?? 50;
return db.listVisionTestRuns(ctx.user.role === "admin" ? undefined : ctx.user.id, limit);
}),
runReference: protectedProcedure
.input(z.object({
referenceImageId: z.number(),
exerciseType: z.string().optional(),
}))
.mutation(async ({ ctx, input }) => {
const reference = await db.getVisionReferenceImageById(input.referenceImageId);
if (!reference || reference.isPublished !== 1) {
throw new Error("Reference image not found");
}
const task = await enqueueTask({
userId: ctx.user.id,
type: "pose_correction_multimodal",
title: `${reference.title} 视觉测试`,
message: "视觉标准图测试已加入后台队列",
payload: {
poseMetrics: {
referenceSource: "vision_reference_library",
expectedFocus: reference.expectedFocus,
sourcePageUrl: reference.sourcePageUrl,
},
exerciseType: input.exerciseType || reference.exerciseType,
detectedIssues: [],
imageUrls: [reference.imageUrl],
},
});
const runId = await db.createVisionTestRun({
taskId: task.taskId,
userId: ctx.user.id,
referenceImageId: reference.id,
title: `${reference.title} 视觉测试`,
exerciseType: input.exerciseType || reference.exerciseType,
imageUrl: reference.imageUrl,
status: "queued",
visionStatus: "pending",
configuredModel: ENV.llmVisionModel || null,
expectedFocus: reference.expectedFocus,
});
return { taskId: task.taskId, runId };
}),
runAll: protectedProcedure.mutation(async ({ ctx }) => {
const references = await db.listVisionReferenceImages();
const queued: Array<{ taskId: string; referenceImageId: number }> = [];
for (const reference of references) {
const task = await enqueueTask({
userId: ctx.user.id,
type: "pose_correction_multimodal",
title: `${reference.title} 视觉测试`,
message: "视觉标准图测试已加入后台队列",
payload: {
poseMetrics: {
referenceSource: "vision_reference_library",
expectedFocus: reference.expectedFocus,
sourcePageUrl: reference.sourcePageUrl,
},
exerciseType: reference.exerciseType,
detectedIssues: [],
imageUrls: [reference.imageUrl],
},
});
await db.createVisionTestRun({
taskId: task.taskId,
userId: ctx.user.id,
referenceImageId: reference.id,
title: `${reference.title} 视觉测试`,
exerciseType: reference.exerciseType,
imageUrl: reference.imageUrl,
status: "queued",
visionStatus: "pending",
configuredModel: ENV.llmVisionModel || null,
expectedFocus: reference.expectedFocus,
});
queued.push({ taskId: task.taskId, referenceImageId: reference.id });
}
return { count: queued.length, queued };
}),
}),
task: router({
list: protectedProcedure
.input(z.object({ limit: z.number().min(1).max(50).default(20) }).optional())

查看文件

@@ -66,6 +66,33 @@ async function invokeStructured<T>(params: StructuredParams<T>) {
throw lastError instanceof Error ? lastError : new Error("Failed to parse structured LLM response");
}
function contentToPlainText(content: Message["content"]) {
if (typeof content === "string") {
return content;
}
const parts = Array.isArray(content) ? content : [content];
return parts
.map((part) => {
if (typeof part === "string") {
return part;
}
if (part.type === "text") {
return part.text;
}
if (part.type === "image_url") {
return `[image] ${part.image_url.url}`;
}
if (part.type === "file_url") {
return `[file] ${part.file_url.url}`;
}
return "";
})
.filter(Boolean)
.join("\n");
}
function parseDataUrl(input: string) {
const match = input.match(/^data:(.+?);base64,(.+)$/);
if (!match) {
@@ -296,7 +323,7 @@ async function createTextCorrectionResult(payload: {
return {
kind: "analysis_corrections" as const,
corrections: response.choices[0]?.message?.content || "暂无建议",
corrections: contentToPlainText(response.choices[0]?.message?.content || "暂无建议"),
};
}
@@ -347,16 +374,26 @@ async function runMultimodalCorrectionTask(task: NonNullable<TaskRow>) {
},
});
return {
const result = {
kind: "pose_correction_multimodal" as const,
imageUrls: payload.imageUrls,
report,
corrections: renderMultimodalCorrectionMarkdown(report as Parameters<typeof renderMultimodalCorrectionMarkdown>[0]),
visionStatus: "ok" as const,
};
await db.completeVisionTestRun(task.id, {
visionStatus: "ok",
summary: (report as { summary?: string }).summary ?? null,
corrections: result.corrections,
report,
warning: null,
});
return result;
} catch (error) {
const fallback = await createTextCorrectionResult(payload);
return {
const result = {
kind: "pose_correction_multimodal" as const,
imageUrls: payload.imageUrls,
report: null,
@@ -364,6 +401,16 @@ async function runMultimodalCorrectionTask(task: NonNullable<TaskRow>) {
visionStatus: "fallback" as const,
warning: error instanceof Error ? error.message : "Vision model unavailable",
};
await db.completeVisionTestRun(task.id, {
visionStatus: "fallback",
summary: null,
corrections: result.corrections,
report: null,
warning: result.warning,
});
return result;
}
}

查看文件

@@ -25,6 +25,7 @@ async function workOnce() {
} catch (error) {
const message = error instanceof Error ? error.message : "Unknown background task error";
await db.failBackgroundTask(task.id, message);
await db.failVisionTestRun(task.id, message);
console.error(`[worker] task ${task.id} failed:`, error);
}