diff --git a/.env.example b/.env.example index 3292757..62503e1 100644 --- a/.env.example +++ b/.env.example @@ -6,6 +6,7 @@ JWT_SECRET=replace-with-strong-secret VITE_APP_ID=tennis-training-hub OAUTH_SERVER_URL= OWNER_OPEN_ID= +ADMIN_USERNAMES=H1 BUILT_IN_FORGE_API_URL= BUILT_IN_FORGE_API_KEY= VITE_OAUTH_PORTAL_URL= diff --git a/README.md b/README.md index a57754a..1682001 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,8 @@ - 图片类任务可单独指定 `LLM_VISION_API_URL` / `LLM_VISION_API_KEY` / `LLM_VISION_MODEL` - 所有图片输入都要求可从公网访问，因此本地相对路径会通过 `APP_PUBLIC_BASE_URL` 规范化为绝对 URL - 若视觉模型链路不可用，系统会自动回退到结构化指标驱动的文本纠正，避免任务直接失败 +- 系统内置“视觉标准图库”页面 `/vision-lab`，可把公网网球参考图入库并保存每次识别结果 +- `ADMIN_USERNAMES` 可指定哪些用户名账号拥有 admin 视角，例如 `H1` ## Quick Start @@ -114,6 +116,7 @@ pnpm exec playwright install chromium - `DATABASE_URL` - `JWT_SECRET` +- `ADMIN_USERNAMES` - `MYSQL_DATABASE` - `MYSQL_USER` - `MYSQL_PASSWORD` diff --git a/client/src/App.tsx b/client/src/App.tsx index 781275d..121c2e4 100644 --- a/client/src/App.tsx +++ b/client/src/App.tsx @@ -19,6 +19,7 @@ import LiveCamera from "./pages/LiveCamera"; import Recorder from "./pages/Recorder"; import Tutorials from "./pages/Tutorials"; import Reminders from "./pages/Reminders"; +import VisionLab from "./pages/VisionLab"; function DashboardRoute({ component: Component }: { component: React.ComponentType }) { return ( @@ -69,6 +70,9 @@ function Router() { + + + diff --git a/client/src/components/DashboardLayout.tsx b/client/src/components/DashboardLayout.tsx index 1730f83..bcf8ae7 100644 --- a/client/src/components/DashboardLayout.tsx +++ b/client/src/components/DashboardLayout.tsx @@ -23,14 +23,22 @@ import { useIsMobile } from "@/hooks/useMobile"; import { LayoutDashboard, LogOut, PanelLeft, Target, Video, Award, Activity, FileVideo, Trophy, Flame, Camera, CircleDot, - BookOpen, Bell + BookOpen, Bell, Microscope } from "lucide-react"; import { CSSProperties, useEffect, useRef, useState } from "react"; import { useLocation, Redirect } from "wouter"; import { DashboardLayoutSkeleton } from './DashboardLayoutSkeleton'; import { TaskCenter } from "./TaskCenter"; -const menuItems = [ +type MenuItem = { + icon: typeof LayoutDashboard; + label: string; + path: string; + group: "main" | "analysis" | "stats" | "learn"; + adminOnly?: boolean; +}; + +const menuItems: MenuItem[] = [ { icon: LayoutDashboard, label: "仪表盘", path: "/dashboard", group: "main" }, { icon: Target, label: "训练计划", path: "/training", group: "main" }, { icon: Flame, label: "每日打卡", path: "/checkin", group: "main" }, @@ -43,6 +51,7 @@ const menuItems = [ { icon: Trophy, label: "排行榜", path: "/leaderboard", group: "stats" }, { icon: BookOpen, label: "教程库", path: "/tutorials", group: "learn" }, { icon: Bell, label: "训练提醒", path: "/reminders", group: "learn" }, + { icon: Microscope, label: "视觉测试", path: "/vision-lab", group: "learn", adminOnly: true }, ]; const mobileNavItems = [ @@ -111,7 +120,8 @@ function DashboardLayoutContent({ const isCollapsed = state === "collapsed"; const [isResizing, setIsResizing] = useState(false); const sidebarRef = useRef(null); - const activeMenuItem = menuItems.find(item => item.path === location); + const visibleMenuItems = menuItems.filter(item => !item.adminOnly || user?.role === "admin"); + const activeMenuItem = visibleMenuItems.find(item => item.path === location); const isMobile = useIsMobile(); useEffect(() => { @@ -180,7 +190,7 @@ function DashboardLayoutContent({ {/* Main group */} - {menuItems.filter(i => i.group === "main").map(item => { + {visibleMenuItems.filter(i => i.group === "main").map(item => { const isActive = location === item.path; return ( @@ -201,7 +211,7 @@ function DashboardLayoutContent({ {!isCollapsed &&
} {!isCollapsed &&

分析与录制

} - {menuItems.filter(i => i.group === "analysis").map(item => { + {visibleMenuItems.filter(i => i.group === "analysis").map(item => { const isActive = location === item.path; return ( @@ -222,7 +232,7 @@ function DashboardLayoutContent({ {!isCollapsed &&
} {!isCollapsed &&

统计与排名

} - {menuItems.filter(i => i.group === "stats").map(item => { + {visibleMenuItems.filter(i => i.group === "stats").map(item => { const isActive = location === item.path; return ( @@ -243,7 +253,7 @@ function DashboardLayoutContent({ {!isCollapsed &&
} {!isCollapsed &&

学习与提醒

} - {menuItems.filter(i => i.group === "learn").map(item => { + {visibleMenuItems.filter(i => i.group === "learn").map(item => { const isActive = location === item.path; return ( diff --git a/client/src/pages/VisionLab.tsx b/client/src/pages/VisionLab.tsx new file mode 100644 index 0000000..c8b2a9d --- /dev/null +++ b/client/src/pages/VisionLab.tsx @@ -0,0 +1,300 @@ +import { useEffect, useMemo, useState } from "react"; +import { useAuth } from "@/_core/hooks/useAuth"; +import { trpc } from "@/lib/trpc"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Button } from "@/components/ui/button"; +import { Badge } from "@/components/ui/badge"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; +import { Skeleton } from "@/components/ui/skeleton"; +import { toast } from "sonner"; +import { useBackgroundTask } from "@/hooks/useBackgroundTask"; +import { Database, Image as ImageIcon, Loader2, Microscope, ShieldCheck, Sparkles } from "lucide-react"; + +type ReferenceImage = { + id: number; + slug: string; + title: string; + exerciseType: string; + imageUrl: string; + sourcePageUrl: string; + sourceLabel: string; + author: string | null; + license: string | null; + expectedFocus: string[] | null; + tags: string[] | null; + notes: string | null; +}; + +type VisionRun = { + id: number; + taskId: string; + userId: number; + userName: string | null; + referenceImageId: number | null; + referenceTitle: string | null; + title: string; + exerciseType: string; + imageUrl: string; + status: "queued" | "succeeded" | "failed"; + visionStatus: "pending" | "ok" | "fallback" | "failed"; + configuredModel: string | null; + expectedFocus: string[] | null; + summary: string | null; + corrections: string | null; + warning: string | null; + error: string | null; + createdAt: Date; + updatedAt: Date; +}; + +function statusBadge(run: VisionRun) { + if (run.status === "failed" || run.visionStatus === "failed") { + return 失败; + } + if (run.status === "queued" || run.visionStatus === "pending") { + return 排队中; + } + if (run.visionStatus === "fallback") { + return 文本降级; + } + return 视觉成功; +} + +export default function VisionLab() { + const { user } = useAuth(); + const utils = trpc.useUtils(); + const [activeTaskId, setActiveTaskId] = useState(null); + const activeTask = useBackgroundTask(activeTaskId); + + const libraryQuery = trpc.vision.library.useQuery(); + const runsQuery = trpc.vision.runs.useQuery( + { limit: 50 }, + { refetchInterval: 4000 } + ); + + const seedMutation = trpc.vision.seedLibrary.useMutation({ + onSuccess: (data) => { + toast.success(`标准图库已就绪，共 ${data.count} 张`); + utils.vision.library.invalidate(); + }, + onError: (error) => toast.error(`标准图库初始化失败: ${error.message}`), + }); + + const runReferenceMutation = trpc.vision.runReference.useMutation({ + onSuccess: (data) => { + setActiveTaskId(data.taskId); + toast.success("视觉测试任务已提交"); + utils.vision.runs.invalidate(); + }, + onError: (error) => toast.error(`视觉测试提交失败: ${error.message}`), + }); + + const runAllMutation = trpc.vision.runAll.useMutation({ + onSuccess: (data) => { + toast.success(`已提交 ${data.count} 个视觉测试任务`); + if (data.queued[0]?.taskId) { + setActiveTaskId(data.queued[0].taskId); + } + utils.vision.runs.invalidate(); + }, + onError: (error) => toast.error(`批量视觉测试提交失败: ${error.message}`), + }); + + useEffect(() => { + if (activeTask.data?.status === "succeeded" || activeTask.data?.status === "failed") { + utils.vision.runs.invalidate(); + setActiveTaskId(null); + } + }, [activeTask.data, utils.vision.runs]); + + const references = useMemo(() => (libraryQuery.data ?? []) as ReferenceImage[], [libraryQuery.data]); + const runs = useMemo(() => (runsQuery.data ?? []) as VisionRun[], [runsQuery.data]); + + if (libraryQuery.isLoading && runsQuery.isLoading) { + return ( +
+ + + +
+ ); + } + + return ( +
+
+
+

视觉标准图库

+

+ 用公网可访问的网球标准图验证多模态纠正链路，并持久化每次测试结果。 +

+
+
+ {user?.role === "admin" ? ( + + ) : null} + +
+
+ + {user?.role === "admin" ? ( + + + Admin 视角 + + 当前账号可查看全部视觉测试记录。若用户名为 `H1` 且被配置进 `ADMIN_USERNAMES`，登录后会自动拥有此视角。 + + + ) : ( + + + 个人测试视角 + 当前页面展示标准图库，以及你自己提交的视觉测试结果。 + + )} + + {activeTask.data?.status === "queued" || activeTask.data?.status === "running" ? ( + + + 后台执行中 + {activeTask.data.message || "视觉测试正在后台执行。"} + + ) : null} + +
+
+

标准图片库

+ {references.length} 张 +
+
+ {references.map((reference) => ( + +
+ {reference.title} +
+ + {reference.title} + + {reference.exerciseType} + {reference.license ? {reference.license} : null} + + + + {reference.notes ? ( +

{reference.notes}

+ ) : null} + {reference.expectedFocus?.length ? ( +
+ {reference.expectedFocus.map((item) => ( + {item} + ))} +
+ ) : null} +
+ + 来源页 + + +
+ + + ))} +
+
+ +
+
+

视觉测试记录

+ {runs.length} 条 +
+
+ {runs.map((run) => ( + + +
+
+
+

{run.title}

+ {statusBadge(run)} + {run.exerciseType} +
+

+ {new Date(run.createdAt).toLocaleString("zh-CN")} + {user?.role === "admin" && run.userName ? ` · 提交人：${run.userName}` : ""} +

+
+ {run.configuredModel ? ( + {run.configuredModel} + ) : null} +
+ + {run.summary ?

{run.summary}

: null} + {run.warning ? ( +

降级说明：{run.warning}

+ ) : null} + {run.error ? ( +

错误：{run.error}

+ ) : null} + + {run.expectedFocus?.length ? ( +
+ {run.expectedFocus.map((item) => ( + {item} + ))} +
+ ) : null} + + {run.corrections ? ( +
+ {run.corrections} +
+ ) : null} + + + ))} + {runs.length === 0 ? ( + + + 还没有视觉测试记录。先运行一张标准图测试，结果会自动入库并显示在这里。 + + + ) : null} +
+
+
+ ); +} diff --git a/docs/FEATURES.md b/docs/FEATURES.md index c2e335f..7fb63d7 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -20,6 +20,7 @@ - 实时摄像头分析：浏览器端调用 MediaPipe，进行姿势识别和反馈展示 - 动作纠正：支持文本纠正和多模态纠正两条链路，统一通过后台任务执行 - 多模态图片输入：上传关键帧后会转换为公网可访问的绝对 URL，再提交给视觉模型 +- 视觉标准图库：内置网球公网参考图，可直接发起视觉识别测试并保存结果 - 视频库：集中展示录制结果、上传结果和分析摘要 ### 在线录制与媒体链路 @@ -51,6 +52,7 @@ - 统一工作台导航 - 仪表盘、训练、视频、录制、分析等模块一致的布局结构 - 全局任务中心：桌面侧边栏和移动端头部都可查看后台任务 +- Admin 视觉测试页：`H1` 这类 admin 用户可查看全部视觉测试数据 - 为后续 PC 粗剪时间线预留媒体域与文档规范 ## 架构能力 diff --git a/docs/testing.md b/docs/testing.md index c730f33..5ffbd9b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -21,6 +21,7 @@ - Node/tRPC 路由输入校验与权限检查 - LLM 模块请求配置与环境变量回退逻辑 - 视觉模型 per-request model override 能力 +- 视觉标准图库路由与 admin/H1 全量可见性逻辑 - 媒体工具函数，例如录制时长格式化与码率选择 ### 3. Go 媒体服务测试 @@ -85,6 +86,14 @@ pnpm exec tsx -e 'import "dotenv/config"; import { invokeLLM } from "./server/_c 如果返回模型与 `LLM_VISION_MODEL` 不一致，说明上游网关忽略了视觉模型选择，业务任务会自动回退到文本纠正结果。 +视觉标准图库的真实 smoke test 可直接复用内置数据： + +- 初始化 `ADMIN_USERNAMES=H1` +- 登录 `H1` 后访问 `/vision-lab` +- 检查标准图是否已经入库 +- 运行单张或批量测试，确认结果会写入 `vision_test_runs` +- 若上游视觉网关不可用，记录应显示 `fallback` + ## Production smoke checks 部署到宿主机后，建议至少补以下联测： diff --git a/docs/verified-features.md b/docs/verified-features.md index a630232..db5c049 100644 --- a/docs/verified-features.md +++ b/docs/verified-features.md @@ -1,18 +1,19 @@ # Verified Features -本文档记录当前已经通过自动化验证或构建验证的项目。更新时间：2026-03-15 00:11 CST。 +本文档记录当前已经通过自动化验证或构建验证的项目。更新时间：2026-03-15 00:40 CST。 ## 最新完整验证记录 - 通过命令：`pnpm verify` -- 验证时间：2026-03-15 00:10 CST -- 结果摘要：`pnpm check` 通过，`pnpm test` 通过（80/80），`pnpm test:go` 通过，`pnpm build` 通过，`pnpm test:e2e` 通过（6/6），`pnpm test:llm` 通过 +- 验证时间：2026-03-15 00:39 CST +- 结果摘要：`pnpm check` 通过，`pnpm test` 通过（85/85），`pnpm test:go` 通过，`pnpm build` 通过，`pnpm test:e2e` 通过（6/6），`pnpm test:llm` 通过 ## 生产部署联测 | 项目 | 验证方式 | 状态 | |------|----------|------| | `https://te.hao.work/` HTTPS 访问 | `curl -I https://te.hao.work/` | 通过 | +| `https://te.hao.work/vision-lab` 视觉测试页访问 | `curl -I https://te.hao.work/vision-lab` | 通过 | | `http://te.hao.work:8302/` 4 位端口访问 | `curl -I http://te.hao.work:8302/` | 通过 | | 站点 TLS 证书 | Let’s Encrypt ECDSA 证书已签发并由宿主机 nginx 加载 | 通过 | | 生产登录与首次进入工作台 | Playwright 登录真实站点并跳转 `/dashboard` | 通过 | @@ -79,6 +80,9 @@ | `.env` 中的 `LLM_API_URL` / `LLM_API_KEY` / `LLM_MODEL` | `pnpm test:llm` | 通过 | | `https://one.hao.work/v1/chat/completions` 联通性 | `pnpm test:llm` 实际返回文本 | 通过 | | 视觉模型独立配置路径 | `server/_core/llm.test.ts` + 手工 smoke 检查 | 通过 | +| 视觉标准图库入库 | MySQL 中 `vision_reference_images` 已写入 5 张 Commons 网球参考图 | 通过 | +| 视觉测试结果入库 | MySQL 中 `vision_test_runs` 已写入 3 条真实测试结果 | 通过 | +| H1 全量可见性 | `H1` 用户已提升为 `admin`，可读取全部视觉测试记录；Playwright 真实站点检查通过 | 通过 | ## 已知非阻断警告 @@ -86,6 +90,7 @@ - `pnpm build` 仍有 Vite 大 chunk 警告；当前属于性能优化待办，不影响本次产物生成 - Playwright 运行依赖 mocked media/network，不等价于真机摄像头、真实弱网和真实 WebRTC 质量验收 - 当前上游视觉网关可能忽略 `LLM_VISION_MODEL` 并回退为文本模型；服务端已实现自动降级，任务不会因此直接失败 +- 2026-03-15 的真实标准图测试中，正手 / 反手 / 发球三条记录均以 `fallback` 完成，说明当前上游视觉网关仍未稳定返回结构化视觉结果 ## 当前未纳入自动验证的内容 diff --git a/drizzle/0006_solid_vision_library.sql b/drizzle/0006_solid_vision_library.sql new file mode 100644 index 0000000..682a225 --- /dev/null +++ b/drizzle/0006_solid_vision_library.sql @@ -0,0 +1,43 @@ +CREATE TABLE `vision_reference_images` ( + `id` int AUTO_INCREMENT NOT NULL, + `slug` varchar(128) NOT NULL, + `title` varchar(256) NOT NULL, + `exerciseType` varchar(64) NOT NULL, + `imageUrl` text NOT NULL, + `sourcePageUrl` text NOT NULL, + `sourceLabel` varchar(128) NOT NULL, + `author` varchar(128), + `license` varchar(128), + `expectedFocus` json, + `tags` json, + `notes` text, + `sortOrder` int NOT NULL DEFAULT 0, + `isPublished` int NOT NULL DEFAULT 1, + `createdAt` timestamp NOT NULL DEFAULT (now()), + `updatedAt` timestamp NOT NULL DEFAULT (now()) ON UPDATE CURRENT_TIMESTAMP, + CONSTRAINT `vision_reference_images_id` PRIMARY KEY(`id`), + CONSTRAINT `vision_reference_images_slug_unique` UNIQUE(`slug`) +); +--> statement-breakpoint +CREATE TABLE `vision_test_runs` ( + `id` int AUTO_INCREMENT NOT NULL, + `taskId` varchar(64) NOT NULL, + `userId` int NOT NULL, + `referenceImageId` int, + `title` varchar(256) NOT NULL, + `exerciseType` varchar(64) NOT NULL, + `imageUrl` text NOT NULL, + `status` enum('queued','succeeded','failed') NOT NULL DEFAULT 'queued', + `visionStatus` enum('pending','ok','fallback','failed') NOT NULL DEFAULT 'pending', + `configuredModel` varchar(128), + `expectedFocus` json, + `summary` text, + `corrections` text, + `report` json, + `warning` text, + `error` text, + `createdAt` timestamp NOT NULL DEFAULT (now()), + `updatedAt` timestamp NOT NULL DEFAULT (now()) ON UPDATE CURRENT_TIMESTAMP, + CONSTRAINT `vision_test_runs_id` PRIMARY KEY(`id`), + CONSTRAINT `vision_test_runs_taskId_unique` UNIQUE(`taskId`) +); diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 6ec2fc8..544eb7a 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -43,6 +43,13 @@ "when": 1773504000000, "tag": "0005_lively_taskmaster", "breakpoints": true + }, + { + "idx": 6, + "version": "5", + "when": 1773510000000, + "tag": "0006_solid_vision_library", + "breakpoints": true } ] } diff --git a/drizzle/schema.ts b/drizzle/schema.ts index 49647b5..c39a637 100644 --- a/drizzle/schema.ts +++ b/drizzle/schema.ts @@ -334,3 +334,55 @@ export const backgroundTasks = mysqlTable("background_tasks", { export type BackgroundTask = typeof backgroundTasks.$inferSelect; export type InsertBackgroundTask = typeof backgroundTasks.$inferInsert; + +/** + * Vision reference library - canonical public tennis images used for multimodal evaluation + */ +export const visionReferenceImages = mysqlTable("vision_reference_images", { + id: int("id").autoincrement().primaryKey(), + slug: varchar("slug", { length: 128 }).notNull().unique(), + title: varchar("title", { length: 256 }).notNull(), + exerciseType: varchar("exerciseType", { length: 64 }).notNull(), + imageUrl: text("imageUrl").notNull(), + sourcePageUrl: text("sourcePageUrl").notNull(), + sourceLabel: varchar("sourceLabel", { length: 128 }).notNull(), + author: varchar("author", { length: 128 }), + license: varchar("license", { length: 128 }), + expectedFocus: json("expectedFocus"), + tags: json("tags"), + notes: text("notes"), + sortOrder: int("sortOrder").default(0).notNull(), + isPublished: int("isPublished").default(1).notNull(), + createdAt: timestamp("createdAt").defaultNow().notNull(), + updatedAt: timestamp("updatedAt").defaultNow().onUpdateNow().notNull(), +}); + +export type VisionReferenceImage = typeof visionReferenceImages.$inferSelect; +export type InsertVisionReferenceImage = typeof visionReferenceImages.$inferInsert; + +/** + * Vision test run history - records each multimodal evaluation against the standard library + */ +export const visionTestRuns = mysqlTable("vision_test_runs", { + id: int("id").autoincrement().primaryKey(), + taskId: varchar("taskId", { length: 64 }).notNull().unique(), + userId: int("userId").notNull(), + referenceImageId: int("referenceImageId"), + title: varchar("title", { length: 256 }).notNull(), + exerciseType: varchar("exerciseType", { length: 64 }).notNull(), + imageUrl: text("imageUrl").notNull(), + status: mysqlEnum("status", ["queued", "succeeded", "failed"]).default("queued").notNull(), + visionStatus: mysqlEnum("visionStatus", ["pending", "ok", "fallback", "failed"]).default("pending").notNull(), + configuredModel: varchar("configuredModel", { length: 128 }), + expectedFocus: json("expectedFocus"), + summary: text("summary"), + corrections: text("corrections"), + report: json("report"), + warning: text("warning"), + error: text("error"), + createdAt: timestamp("createdAt").defaultNow().notNull(), + updatedAt: timestamp("updatedAt").defaultNow().onUpdateNow().notNull(), +}); + +export type VisionTestRun = typeof visionTestRuns.$inferSelect; +export type InsertVisionTestRun = typeof visionTestRuns.$inferInsert; diff --git a/server/_core/env.ts b/server/_core/env.ts index 0529d10..cbeedc5 100644 --- a/server/_core/env.ts +++ b/server/_core/env.ts @@ -9,6 +9,12 @@ const parseBoolean = (value: string | undefined, fallback: boolean) => { return value === "1" || value.toLowerCase() === "true"; }; +const parseList = (value: string | undefined) => + (value ?? "") + .split(",") + .map((item) => item.trim()) + .filter(Boolean); + export const ENV = { appId: process.env.VITE_APP_ID ?? "", appPublicBaseUrl: process.env.APP_PUBLIC_BASE_URL ?? "", @@ -16,6 +22,7 @@ export const ENV = { databaseUrl: process.env.DATABASE_URL ?? "", oAuthServerUrl: process.env.OAUTH_SERVER_URL ?? "", ownerOpenId: process.env.OWNER_OPEN_ID ?? "", + adminUsernames: parseList(process.env.ADMIN_USERNAMES), isProduction: process.env.NODE_ENV === "production", forgeApiUrl: process.env.BUILT_IN_FORGE_API_URL ?? "", forgeApiKey: process.env.BUILT_IN_FORGE_API_KEY ?? "", diff --git a/server/_core/index.ts b/server/_core/index.ts index 7841c7d..e6024fd 100644 --- a/server/_core/index.ts +++ b/server/_core/index.ts @@ -9,6 +9,7 @@ import { appRouter } from "../routers"; import { createContext } from "./context"; import { registerMediaProxy } from "./mediaProxy"; import { serveStatic } from "./static"; +import { seedTutorials, seedVisionReferenceImages } from "../db"; function isPortAvailable(port: number): Promise { return new Promise(resolve => { @@ -30,6 +31,9 @@ async function findAvailablePort(startPort: number = 3000): Promise { } async function startServer() { + await seedTutorials(); + await seedVisionReferenceImages(); + const app = express(); const server = createServer(app); registerMediaProxy(app); diff --git a/server/db.ts b/server/db.ts index 0708c9b..a8435de 100644 --- a/server/db.ts +++ b/server/db.ts @@ -15,6 +15,8 @@ import { trainingReminders, InsertTrainingReminder, notificationLog, InsertNotificationLog, backgroundTasks, InsertBackgroundTask, + visionReferenceImages, InsertVisionReferenceImage, + visionTestRuns, InsertVisionTestRun, } from "../drizzle/schema"; import { ENV } from './_core/env'; @@ -89,8 +91,9 @@ export async function createUsernameAccount(username: string): Promise<{ user: t if (existing.length > 0) { const user = await db.select().from(users).where(eq(users.id, existing[0].userId)).limit(1); if (user.length > 0) { - await db.update(users).set({ lastSignedIn: new Date() }).where(eq(users.id, user[0].id)); - return { user: user[0], isNew: false }; + const updatedRole = ENV.adminUsernames.includes(username) ? "admin" : user[0].role; + await db.update(users).set({ lastSignedIn: new Date(), role: updatedRole }).where(eq(users.id, user[0].id)); + return { user: { ...user[0], role: updatedRole, lastSignedIn: new Date() }, isNew: false }; } } @@ -100,6 +103,7 @@ export async function createUsernameAccount(username: string): Promise<{ user: t openId, name: username, loginMethod: "username", + role: ENV.adminUsernames.includes(username) ? "admin" : "user", lastSignedIn: new Date(), ntrpRating: 1.5, totalSessions: 0, @@ -443,6 +447,223 @@ export async function getLeaderboard(sortBy: "ntrpRating" | "totalMinutes" | "to }).from(users).orderBy(desc(sortColumn)).limit(limit); } +// ===== VISION REFERENCE LIBRARY ===== + +export const VISION_REFERENCE_SEED_DATA: Omit< + InsertVisionReferenceImage, + "id" | "createdAt" | "updatedAt" +>[] = [ + { + slug: "commons-forehand-tennispictures", + title: "标准图：正手挥拍", + exerciseType: "forehand", + imageUrl: "https://commons.wikimedia.org/wiki/Special:FilePath/Forehand.jpg", + sourcePageUrl: "https://commons.wikimedia.org/wiki/File:Forehand.jpg", + sourceLabel: "Wikimedia Commons", + author: "Tennispictures", + license: "CC0 1.0", + expectedFocus: ["引拍完整", "击球臂路径", "肩髋转动", "重心转移"], + tags: ["forehand", "reference", "commons", "stroke"], + notes: "用于检测模型对正手引拍、发力和随挥阶段的描述能力。", + sortOrder: 1, + isPublished: 1, + }, + { + slug: "commons-backhand-federer", + title: "标准图：反手挥拍", + exerciseType: "backhand", + imageUrl: "https://commons.wikimedia.org/wiki/Special:FilePath/Backhand_Federer.jpg", + sourcePageUrl: "https://commons.wikimedia.org/wiki/File:Backhand_Federer.jpg", + sourceLabel: "Wikimedia Commons", + author: "Ian Gampon", + license: "CC BY 2.0", + expectedFocus: ["非持拍手收回", "躯干旋转", "拍面路径", "击球点位置"], + tags: ["backhand", "reference", "commons", "stroke"], + notes: "用于检测模型对单反/反手击球阶段和身体协同的判断。", + sortOrder: 2, + isPublished: 1, + }, + { + slug: "commons-serena-serve", + title: "标准图：发球", + exerciseType: "serve", + imageUrl: "https://commons.wikimedia.org/wiki/Special:FilePath/Serena_Williams_Serves.JPG", + sourcePageUrl: "https://commons.wikimedia.org/wiki/File:Serena_Williams_Serves.JPG", + sourceLabel: "Wikimedia Commons", + author: "Clavecin", + license: "Public Domain", + expectedFocus: ["抛球与击球点", "肩肘链条", "躯干伸展", "落地重心"], + tags: ["serve", "reference", "commons", "overhead"], + notes: "用于检测模型对发球上举、鞭打和击球点的识别能力。", + sortOrder: 3, + isPublished: 1, + }, + { + slug: "commons-volley-lewis", + title: "标准图：网前截击", + exerciseType: "volley", + imageUrl: "https://commons.wikimedia.org/wiki/Special:FilePath/Ernest_w._lewis,_volleying.jpg", + sourcePageUrl: "https://commons.wikimedia.org/wiki/File:Ernest_w._lewis,_volleying.jpg", + sourceLabel: "Wikimedia Commons", + author: "Unknown author", + license: "Public Domain", + expectedFocus: ["拍头稳定", "准备姿态", "身体前压", "短促触球"], + tags: ["volley", "reference", "commons", "net-play"], + notes: "用于检测模型对截击站位和紧凑击球结构的识别能力。", + sortOrder: 4, + isPublished: 1, + }, + { + slug: "commons-tiafoe-backhand", + title: "标准图：现代反手参考", + exerciseType: "backhand", + imageUrl: "https://commons.wikimedia.org/wiki/Special:FilePath/Frances_Tiafoe_Backhand.jpg", + sourcePageUrl: "https://commons.wikimedia.org/wiki/File:Frances_Tiafoe_Backhand.jpg", + sourceLabel: "Wikimedia Commons", + author: null, + license: "Wikimedia Commons file license", + expectedFocus: ["双手协同", "脚步支撑", "肩髋分离", "随挥方向"], + tags: ["backhand", "reference", "commons", "modern"], + notes: "补充现代职业选手反手样本，便于比较传统与现代动作语言。", + sortOrder: 5, + isPublished: 1, + }, +]; + +export async function seedVisionReferenceImages() { + const db = await getDb(); + if (!db) return; + + const existing = await db.select().from(visionReferenceImages).limit(1); + if (existing.length > 0) return; + + for (const item of VISION_REFERENCE_SEED_DATA) { + await db.insert(visionReferenceImages).values(item); + } +} + +export async function listVisionReferenceImages() { + const db = await getDb(); + if (!db) return []; + + return db.select().from(visionReferenceImages) + .where(eq(visionReferenceImages.isPublished, 1)) + .orderBy(asc(visionReferenceImages.sortOrder), asc(visionReferenceImages.id)); +} + +export async function getVisionReferenceImageById(id: number) { + const db = await getDb(); + if (!db) return undefined; + + const result = await db.select().from(visionReferenceImages) + .where(eq(visionReferenceImages.id, id)) + .limit(1); + return result[0]; +} + +export async function createVisionTestRun(run: InsertVisionTestRun) { + const db = await getDb(); + if (!db) throw new Error("Database not available"); + const result = await db.insert(visionTestRuns).values(run); + return result[0].insertId; +} + +export async function listVisionTestRuns(userId?: number, limit = 50) { + const db = await getDb(); + if (!db) return []; + + const query = db.select({ + id: visionTestRuns.id, + taskId: visionTestRuns.taskId, + userId: visionTestRuns.userId, + userName: users.name, + referenceImageId: visionTestRuns.referenceImageId, + referenceTitle: visionReferenceImages.title, + title: visionTestRuns.title, + exerciseType: visionTestRuns.exerciseType, + imageUrl: visionTestRuns.imageUrl, + status: visionTestRuns.status, + visionStatus: visionTestRuns.visionStatus, + configuredModel: visionTestRuns.configuredModel, + expectedFocus: visionTestRuns.expectedFocus, + summary: visionTestRuns.summary, + corrections: visionTestRuns.corrections, + report: visionTestRuns.report, + warning: visionTestRuns.warning, + error: visionTestRuns.error, + createdAt: visionTestRuns.createdAt, + updatedAt: visionTestRuns.updatedAt, + }).from(visionTestRuns) + .leftJoin(users, eq(users.id, visionTestRuns.userId)) + .leftJoin(visionReferenceImages, eq(visionReferenceImages.id, visionTestRuns.referenceImageId)) + .orderBy(desc(visionTestRuns.createdAt)) + .limit(limit); + + if (userId == null) { + return query; + } + + return db.select({ + id: visionTestRuns.id, + taskId: visionTestRuns.taskId, + userId: visionTestRuns.userId, + userName: users.name, + referenceImageId: visionTestRuns.referenceImageId, + referenceTitle: visionReferenceImages.title, + title: visionTestRuns.title, + exerciseType: visionTestRuns.exerciseType, + imageUrl: visionTestRuns.imageUrl, + status: visionTestRuns.status, + visionStatus: visionTestRuns.visionStatus, + configuredModel: visionTestRuns.configuredModel, + expectedFocus: visionTestRuns.expectedFocus, + summary: visionTestRuns.summary, + corrections: visionTestRuns.corrections, + report: visionTestRuns.report, + warning: visionTestRuns.warning, + error: visionTestRuns.error, + createdAt: visionTestRuns.createdAt, + updatedAt: visionTestRuns.updatedAt, + }).from(visionTestRuns) + .leftJoin(users, eq(users.id, visionTestRuns.userId)) + .leftJoin(visionReferenceImages, eq(visionReferenceImages.id, visionTestRuns.referenceImageId)) + .where(eq(visionTestRuns.userId, userId)) + .orderBy(desc(visionTestRuns.createdAt)) + .limit(limit); +} + +export async function completeVisionTestRun(taskId: string, data: { + visionStatus: "ok" | "fallback"; + summary?: string | null; + corrections: string; + report?: unknown; + warning?: string | null; +}) { + const db = await getDb(); + if (!db) return; + + await db.update(visionTestRuns).set({ + status: "succeeded", + visionStatus: data.visionStatus, + summary: data.summary ?? null, + corrections: data.corrections, + report: data.report ?? null, + warning: data.warning ?? null, + error: null, + }).where(eq(visionTestRuns.taskId, taskId)); +} + +export async function failVisionTestRun(taskId: string, error: string) { + const db = await getDb(); + if (!db) return; + + await db.update(visionTestRuns).set({ + status: "failed", + visionStatus: "failed", + error, + }).where(eq(visionTestRuns.taskId, taskId)); +} + // ===== TUTORIAL OPERATIONS ===== export const TUTORIAL_SEED_DATA: Omit[] = [ diff --git a/server/features.test.ts b/server/features.test.ts index ec55c77..2e85dc2 100644 --- a/server/features.test.ts +++ b/server/features.test.ts @@ -816,3 +816,80 @@ describe("notification.markAllRead", () => { await expect(caller.notification.markAllRead()).rejects.toThrow(); }); }); + +// ===== VISION LIBRARY TESTS ===== + +describe("vision.library", () => { + it("requires authentication", async () => { + const { ctx } = createMockContext(null); + const caller = appRouter.createCaller(ctx); + await expect(caller.vision.library()).rejects.toThrow(); + }); + + it("returns seeded references for authenticated users", async () => { + const user = createTestUser(); + const { ctx } = createMockContext(user); + const caller = appRouter.createCaller(ctx); + const seedSpy = vi.spyOn(db, "seedVisionReferenceImages").mockResolvedValueOnce(); + const listSpy = vi.spyOn(db, "listVisionReferenceImages").mockResolvedValueOnce([ + { + id: 1, + slug: "ref-1", + title: "标准图：正手挥拍", + exerciseType: "forehand", + imageUrl: "https://example.com/forehand.jpg", + sourcePageUrl: "https://example.com/source", + sourceLabel: "Example", + author: null, + license: null, + expectedFocus: ["肩髋转动"], + tags: ["forehand"], + notes: null, + sortOrder: 1, + isPublished: 1, + createdAt: new Date(), + updatedAt: new Date(), + }, + ] as any); + + const result = await caller.vision.library(); + + expect(seedSpy).toHaveBeenCalledTimes(1); + expect(listSpy).toHaveBeenCalledTimes(1); + expect(result).toHaveLength(1); + }); +}); + +describe("vision.runs", () => { + it("limits regular users to their own vision test runs", async () => { + const user = createTestUser({ id: 7, role: "user" }); + const { ctx } = createMockContext(user); + const caller = appRouter.createCaller(ctx); + const listSpy = vi.spyOn(db, "listVisionTestRuns").mockResolvedValueOnce([]); + + await caller.vision.runs({ limit: 20 }); + + expect(listSpy).toHaveBeenCalledWith(7, 20); + }); + + it("allows admin users to view all vision test runs", async () => { + const admin = createTestUser({ id: 9, role: "admin", name: "H1" }); + const { ctx } = createMockContext(admin); + const caller = appRouter.createCaller(ctx); + const listSpy = vi.spyOn(db, "listVisionTestRuns").mockResolvedValueOnce([]); + + await caller.vision.runs({ limit: 30 }); + + expect(listSpy).toHaveBeenCalledWith(undefined, 30); + }); +}); + +describe("vision.seedLibrary", () => { + it("rejects non-admin users", async () => { + const user = createTestUser({ role: "user" }); + const { ctx } = createMockContext(user); + const caller = appRouter.createCaller(ctx); + + await expect(caller.vision.seedLibrary()).rejects.toThrow(); + }); +}); diff --git a/server/routers.ts b/server/routers.ts index bf42265..e20f13f 100644 --- a/server/routers.ts +++ b/server/routers.ts @@ -1,9 +1,10 @@ import { COOKIE_NAME, ONE_YEAR_MS } from "@shared/const"; import { getSessionCookieOptions } from "./_core/cookies"; import { systemRouter } from "./_core/systemRouter"; -import { publicProcedure, protectedProcedure, router } from "./_core/trpc"; +import { adminProcedure, publicProcedure, protectedProcedure, router } from "./_core/trpc"; import { z } from "zod"; import { sdk } from "./_core/sdk"; +import { ENV } from "./_core/env"; import { storagePut } from "./storage"; import * as db from "./db"; import { nanoid } from "nanoid"; @@ -271,6 +272,111 @@ export const appRouter = router({ }), }), + vision: router({ + library: protectedProcedure.query(async () => { + await db.seedVisionReferenceImages(); + return db.listVisionReferenceImages(); + }), + + seedLibrary: adminProcedure.mutation(async () => { + await db.seedVisionReferenceImages(); + const images = await db.listVisionReferenceImages(); + return { count: images.length }; + }), + + runs: protectedProcedure + .input(z.object({ limit: z.number().min(1).max(100).default(50) }).optional()) + .query(async ({ ctx, input }) => { + const limit = input?.limit ?? 50; + return db.listVisionTestRuns(ctx.user.role === "admin" ? undefined : ctx.user.id, limit); + }), + + runReference: protectedProcedure + .input(z.object({ + referenceImageId: z.number(), + exerciseType: z.string().optional(), + })) + .mutation(async ({ ctx, input }) => { + const reference = await db.getVisionReferenceImageById(input.referenceImageId); + if (!reference || reference.isPublished !== 1) { + throw new Error("Reference image not found"); + } + + const task = await enqueueTask({ + userId: ctx.user.id, + type: "pose_correction_multimodal", + title: `${reference.title} 视觉测试`, + message: "视觉标准图测试已加入后台队列", + payload: { + poseMetrics: { + referenceSource: "vision_reference_library", + expectedFocus: reference.expectedFocus, + sourcePageUrl: reference.sourcePageUrl, + }, + exerciseType: input.exerciseType || reference.exerciseType, + detectedIssues: [], + imageUrls: [reference.imageUrl], + }, + }); + + const runId = await db.createVisionTestRun({ + taskId: task.taskId, + userId: ctx.user.id, + referenceImageId: reference.id, + title: `${reference.title} 视觉测试`, + exerciseType: input.exerciseType || reference.exerciseType, + imageUrl: reference.imageUrl, + status: "queued", + visionStatus: "pending", + configuredModel: ENV.llmVisionModel || null, + expectedFocus: reference.expectedFocus, + }); + + return { taskId: task.taskId, runId }; + }), + + runAll: protectedProcedure.mutation(async ({ ctx }) => { + const references = await db.listVisionReferenceImages(); + const queued: Array<{ taskId: string; referenceImageId: number }> = []; + + for (const reference of references) { + const task = await enqueueTask({ + userId: ctx.user.id, + type: "pose_correction_multimodal", + title: `${reference.title} 视觉测试`, + message: "视觉标准图测试已加入后台队列", + payload: { + poseMetrics: { + referenceSource: "vision_reference_library", + expectedFocus: reference.expectedFocus, + sourcePageUrl: reference.sourcePageUrl, + }, + exerciseType: reference.exerciseType, + detectedIssues: [], + imageUrls: [reference.imageUrl], + }, + }); + + await db.createVisionTestRun({ + taskId: task.taskId, + userId: ctx.user.id, + referenceImageId: reference.id, + title: `${reference.title} 视觉测试`, + exerciseType: reference.exerciseType, + imageUrl: reference.imageUrl, + status: "queued", + visionStatus: "pending", + configuredModel: ENV.llmVisionModel || null, + expectedFocus: reference.expectedFocus, + }); + + queued.push({ taskId: task.taskId, referenceImageId: reference.id }); + } + + return { count: queued.length, queued }; + }), + }), + task: router({ list: protectedProcedure .input(z.object({ limit: z.number().min(1).max(50).default(20) }).optional()) diff --git a/server/taskWorker.ts b/server/taskWorker.ts index b1e6928..d3705e6 100644 --- a/server/taskWorker.ts +++ b/server/taskWorker.ts @@ -66,6 +66,33 @@ async function invokeStructured(params: StructuredParams) { throw lastError instanceof Error ? lastError : new Error("Failed to parse structured LLM response"); } +function contentToPlainText(content: Message["content"]) { + if (typeof content === "string") { + return content; + } + + const parts = Array.isArray(content) ? content : [content]; + + return parts + .map((part) => { + if (typeof part === "string") { + return part; + } + if (part.type === "text") { + return part.text; + } + if (part.type === "image_url") { + return `[image] ${part.image_url.url}`; + } + if (part.type === "file_url") { + return `[file] ${part.file_url.url}`; + } + return ""; + }) + .filter(Boolean) + .join("\n"); +} + function parseDataUrl(input: string) { const match = input.match(/^data:(.+?);base64,(.+)$/); if (!match) { @@ -296,7 +323,7 @@ async function createTextCorrectionResult(payload: { return { kind: "analysis_corrections" as const, - corrections: response.choices[0]?.message?.content || "暂无建议", + corrections: contentToPlainText(response.choices[0]?.message?.content || "暂无建议"), }; } @@ -347,16 +374,26 @@ async function runMultimodalCorrectionTask(task: NonNullable) { }, }); - return { + const result = { kind: "pose_correction_multimodal" as const, imageUrls: payload.imageUrls, report, corrections: renderMultimodalCorrectionMarkdown(report as Parameters[0]), visionStatus: "ok" as const, }; + + await db.completeVisionTestRun(task.id, { + visionStatus: "ok", + summary: (report as { summary?: string }).summary ?? null, + corrections: result.corrections, + report, + warning: null, + }); + + return result; } catch (error) { const fallback = await createTextCorrectionResult(payload); - return { + const result = { kind: "pose_correction_multimodal" as const, imageUrls: payload.imageUrls, report: null, @@ -364,6 +401,16 @@ async function runMultimodalCorrectionTask(task: NonNullable) { visionStatus: "fallback" as const, warning: error instanceof Error ? error.message : "Vision model unavailable", }; + + await db.completeVisionTestRun(task.id, { + visionStatus: "fallback", + summary: null, + corrections: result.corrections, + report: null, + warning: result.warning, + }); + + return result; } } diff --git a/server/worker.ts b/server/worker.ts index 5a9affa..6438057 100644 --- a/server/worker.ts +++ b/server/worker.ts @@ -25,6 +25,7 @@ async function workOnce() { } catch (error) { const message = error instanceof Error ? error.message : "Unknown background task error"; await db.failBackgroundTask(task.id, message); + await db.failVisionTestRun(task.id, message); console.error(`[worker] task ${task.id} failed:`, error); }