Add admin vision lab and LLM vision verification

2026-03-15 00:41:09 +08:00
--- a/server/_core/env.ts
+++ b/server/_core/env.ts
@@ -9,6 +9,12 @@ const parseBoolean = (value: string | undefined, fallback: boolean) => {
  return value === "1" || value.toLowerCase() === "true";
 };

+const parseList = (value: string | undefined) =>
+  (value ?? "")
+    .split(",")
+    .map((item) => item.trim())
+    .filter(Boolean);
+
 export const ENV = {
  appId: process.env.VITE_APP_ID ?? "",
  appPublicBaseUrl: process.env.APP_PUBLIC_BASE_URL ?? "",
@@ -16,6 +22,7 @@ export const ENV = {
  databaseUrl: process.env.DATABASE_URL ?? "",
  oAuthServerUrl: process.env.OAUTH_SERVER_URL ?? "",
  ownerOpenId: process.env.OWNER_OPEN_ID ?? "",
+  adminUsernames: parseList(process.env.ADMIN_USERNAMES),
  isProduction: process.env.NODE_ENV === "production",
  forgeApiUrl: process.env.BUILT_IN_FORGE_API_URL ?? "",
  forgeApiKey: process.env.BUILT_IN_FORGE_API_KEY ?? "",
--- a/server/_core/index.ts
+++ b/server/_core/index.ts
@@ -9,6 +9,7 @@ import { appRouter } from "../routers";
 import { createContext } from "./context";
 import { registerMediaProxy } from "./mediaProxy";
 import { serveStatic } from "./static";
+import { seedTutorials, seedVisionReferenceImages } from "../db";

 function isPortAvailable(port: number): Promise<boolean> {
  return new Promise(resolve => {
@@ -30,6 +31,9 @@ async function findAvailablePort(startPort: number = 3000): Promise<number> {
 }

 async function startServer() {
+  await seedTutorials();
+  await seedVisionReferenceImages();
+
  const app = express();
  const server = createServer(app);
  registerMediaProxy(app);
--- a/server/db.ts
+++ b/server/db.ts
@@ -15,6 +15,8 @@ import {
  trainingReminders, InsertTrainingReminder,
  notificationLog, InsertNotificationLog,
  backgroundTasks, InsertBackgroundTask,
+  visionReferenceImages, InsertVisionReferenceImage,
+  visionTestRuns, InsertVisionTestRun,
 } from "../drizzle/schema";
 import { ENV } from './_core/env';

@@ -89,8 +91,9 @@ export async function createUsernameAccount(username: string): Promise<{ user: t
  if (existing.length > 0) {
    const user = await db.select().from(users).where(eq(users.id, existing[0].userId)).limit(1);
    if (user.length > 0) {
-      await db.update(users).set({ lastSignedIn: new Date() }).where(eq(users.id, user[0].id));
-      return { user: user[0], isNew: false };
+      const updatedRole = ENV.adminUsernames.includes(username) ? "admin" : user[0].role;
+      await db.update(users).set({ lastSignedIn: new Date(), role: updatedRole }).where(eq(users.id, user[0].id));
+      return { user: { ...user[0], role: updatedRole, lastSignedIn: new Date() }, isNew: false };
    }
  }

@@ -100,6 +103,7 @@ export async function createUsernameAccount(username: string): Promise<{ user: t
    openId,
    name: username,
    loginMethod: "username",
+    role: ENV.adminUsernames.includes(username) ? "admin" : "user",
    lastSignedIn: new Date(),
    ntrpRating: 1.5,
    totalSessions: 0,
@@ -443,6 +447,223 @@ export async function getLeaderboard(sortBy: "ntrpRating" | "totalMinutes" | "to
  }).from(users).orderBy(desc(sortColumn)).limit(limit);
 }

+// ===== VISION REFERENCE LIBRARY =====
+
+export const VISION_REFERENCE_SEED_DATA: Omit<
+  InsertVisionReferenceImage,
+  "id" | "createdAt" | "updatedAt"
+>[] = [
+  {
+    slug: "commons-forehand-tennispictures",
+    title: "标准图：正手挥拍",
+    exerciseType: "forehand",
+    imageUrl: "https://commons.wikimedia.org/wiki/Special:FilePath/Forehand.jpg",
+    sourcePageUrl: "https://commons.wikimedia.org/wiki/File:Forehand.jpg",
+    sourceLabel: "Wikimedia Commons",
+    author: "Tennispictures",
+    license: "CC0 1.0",
+    expectedFocus: ["引拍完整", "击球臂路径", "肩髋转动", "重心转移"],
+    tags: ["forehand", "reference", "commons", "stroke"],
+    notes: "用于检测模型对正手引拍、发力和随挥阶段的描述能力。",
+    sortOrder: 1,
+    isPublished: 1,
+  },
+  {
+    slug: "commons-backhand-federer",
+    title: "标准图：反手挥拍",
+    exerciseType: "backhand",
+    imageUrl: "https://commons.wikimedia.org/wiki/Special:FilePath/Backhand_Federer.jpg",
+    sourcePageUrl: "https://commons.wikimedia.org/wiki/File:Backhand_Federer.jpg",
+    sourceLabel: "Wikimedia Commons",
+    author: "Ian Gampon",
+    license: "CC BY 2.0",
+    expectedFocus: ["非持拍手收回", "躯干旋转", "拍面路径", "击球点位置"],
+    tags: ["backhand", "reference", "commons", "stroke"],
+    notes: "用于检测模型对单反/反手击球阶段和身体协同的判断。",
+    sortOrder: 2,
+    isPublished: 1,
+  },
+  {
+    slug: "commons-serena-serve",
+    title: "标准图：发球",
+    exerciseType: "serve",
+    imageUrl: "https://commons.wikimedia.org/wiki/Special:FilePath/Serena_Williams_Serves.JPG",
+    sourcePageUrl: "https://commons.wikimedia.org/wiki/File:Serena_Williams_Serves.JPG",
+    sourceLabel: "Wikimedia Commons",
+    author: "Clavecin",
+    license: "Public Domain",
+    expectedFocus: ["抛球与击球点", "肩肘链条", "躯干伸展", "落地重心"],
+    tags: ["serve", "reference", "commons", "overhead"],
+    notes: "用于检测模型对发球上举、鞭打和击球点的识别能力。",
+    sortOrder: 3,
+    isPublished: 1,
+  },
+  {
+    slug: "commons-volley-lewis",
+    title: "标准图：网前截击",
+    exerciseType: "volley",
+    imageUrl: "https://commons.wikimedia.org/wiki/Special:FilePath/Ernest_w._lewis,_volleying.jpg",
+    sourcePageUrl: "https://commons.wikimedia.org/wiki/File:Ernest_w._lewis,_volleying.jpg",
+    sourceLabel: "Wikimedia Commons",
+    author: "Unknown author",
+    license: "Public Domain",
+    expectedFocus: ["拍头稳定", "准备姿态", "身体前压", "短促触球"],
+    tags: ["volley", "reference", "commons", "net-play"],
+    notes: "用于检测模型对截击站位和紧凑击球结构的识别能力。",
+    sortOrder: 4,
+    isPublished: 1,
+  },
+  {
+    slug: "commons-tiafoe-backhand",
+    title: "标准图：现代反手参考",
+    exerciseType: "backhand",
+    imageUrl: "https://commons.wikimedia.org/wiki/Special:FilePath/Frances_Tiafoe_Backhand.jpg",
+    sourcePageUrl: "https://commons.wikimedia.org/wiki/File:Frances_Tiafoe_Backhand.jpg",
+    sourceLabel: "Wikimedia Commons",
+    author: null,
+    license: "Wikimedia Commons file license",
+    expectedFocus: ["双手协同", "脚步支撑", "肩髋分离", "随挥方向"],
+    tags: ["backhand", "reference", "commons", "modern"],
+    notes: "补充现代职业选手反手样本，便于比较传统与现代动作语言。",
+    sortOrder: 5,
+    isPublished: 1,
+  },
+];
+
+export async function seedVisionReferenceImages() {
+  const db = await getDb();
+  if (!db) return;
+
+  const existing = await db.select().from(visionReferenceImages).limit(1);
+  if (existing.length > 0) return;
+
+  for (const item of VISION_REFERENCE_SEED_DATA) {
+    await db.insert(visionReferenceImages).values(item);
+  }
+}
+
+export async function listVisionReferenceImages() {
+  const db = await getDb();
+  if (!db) return [];
+
+  return db.select().from(visionReferenceImages)
+    .where(eq(visionReferenceImages.isPublished, 1))
+    .orderBy(asc(visionReferenceImages.sortOrder), asc(visionReferenceImages.id));
+}
+
+export async function getVisionReferenceImageById(id: number) {
+  const db = await getDb();
+  if (!db) return undefined;
+
+  const result = await db.select().from(visionReferenceImages)
+    .where(eq(visionReferenceImages.id, id))
+    .limit(1);
+  return result[0];
+}
+
+export async function createVisionTestRun(run: InsertVisionTestRun) {
+  const db = await getDb();
+  if (!db) throw new Error("Database not available");
+  const result = await db.insert(visionTestRuns).values(run);
+  return result[0].insertId;
+}
+
+export async function listVisionTestRuns(userId?: number, limit = 50) {
+  const db = await getDb();
+  if (!db) return [];
+
+  const query = db.select({
+    id: visionTestRuns.id,
+    taskId: visionTestRuns.taskId,
+    userId: visionTestRuns.userId,
+    userName: users.name,
+    referenceImageId: visionTestRuns.referenceImageId,
+    referenceTitle: visionReferenceImages.title,
+    title: visionTestRuns.title,
+    exerciseType: visionTestRuns.exerciseType,
+    imageUrl: visionTestRuns.imageUrl,
+    status: visionTestRuns.status,
+    visionStatus: visionTestRuns.visionStatus,
+    configuredModel: visionTestRuns.configuredModel,
+    expectedFocus: visionTestRuns.expectedFocus,
+    summary: visionTestRuns.summary,
+    corrections: visionTestRuns.corrections,
+    report: visionTestRuns.report,
+    warning: visionTestRuns.warning,
+    error: visionTestRuns.error,
+    createdAt: visionTestRuns.createdAt,
+    updatedAt: visionTestRuns.updatedAt,
+  }).from(visionTestRuns)
+    .leftJoin(users, eq(users.id, visionTestRuns.userId))
+    .leftJoin(visionReferenceImages, eq(visionReferenceImages.id, visionTestRuns.referenceImageId))
+    .orderBy(desc(visionTestRuns.createdAt))
+    .limit(limit);
+
+  if (userId == null) {
+    return query;
+  }
+
+  return db.select({
+    id: visionTestRuns.id,
+    taskId: visionTestRuns.taskId,
+    userId: visionTestRuns.userId,
+    userName: users.name,
+    referenceImageId: visionTestRuns.referenceImageId,
+    referenceTitle: visionReferenceImages.title,
+    title: visionTestRuns.title,
+    exerciseType: visionTestRuns.exerciseType,
+    imageUrl: visionTestRuns.imageUrl,
+    status: visionTestRuns.status,
+    visionStatus: visionTestRuns.visionStatus,
+    configuredModel: visionTestRuns.configuredModel,
+    expectedFocus: visionTestRuns.expectedFocus,
+    summary: visionTestRuns.summary,
+    corrections: visionTestRuns.corrections,
+    report: visionTestRuns.report,
+    warning: visionTestRuns.warning,
+    error: visionTestRuns.error,
+    createdAt: visionTestRuns.createdAt,
+    updatedAt: visionTestRuns.updatedAt,
+  }).from(visionTestRuns)
+    .leftJoin(users, eq(users.id, visionTestRuns.userId))
+    .leftJoin(visionReferenceImages, eq(visionReferenceImages.id, visionTestRuns.referenceImageId))
+    .where(eq(visionTestRuns.userId, userId))
+    .orderBy(desc(visionTestRuns.createdAt))
+    .limit(limit);
+}
+
+export async function completeVisionTestRun(taskId: string, data: {
+  visionStatus: "ok" | "fallback";
+  summary?: string | null;
+  corrections: string;
+  report?: unknown;
+  warning?: string | null;
+}) {
+  const db = await getDb();
+  if (!db) return;
+
+  await db.update(visionTestRuns).set({
+    status: "succeeded",
+    visionStatus: data.visionStatus,
+    summary: data.summary ?? null,
+    corrections: data.corrections,
+    report: data.report ?? null,
+    warning: data.warning ?? null,
+    error: null,
+  }).where(eq(visionTestRuns.taskId, taskId));
+}
+
+export async function failVisionTestRun(taskId: string, error: string) {
+  const db = await getDb();
+  if (!db) return;
+
+  await db.update(visionTestRuns).set({
+    status: "failed",
+    visionStatus: "failed",
+    error,
+  }).where(eq(visionTestRuns.taskId, taskId));
+}
+
 // ===== TUTORIAL OPERATIONS =====

 export const TUTORIAL_SEED_DATA: Omit<InsertTutorialVideo, "id">[] = [
--- a/server/features.test.ts
+++ b/server/features.test.ts
@@ -816,3 +816,80 @@ describe("notification.markAllRead", () => {
    await expect(caller.notification.markAllRead()).rejects.toThrow();
  });
 });
+
+// ===== VISION LIBRARY TESTS =====
+
+describe("vision.library", () => {
+  it("requires authentication", async () => {
+    const { ctx } = createMockContext(null);
+    const caller = appRouter.createCaller(ctx);
+    await expect(caller.vision.library()).rejects.toThrow();
+  });
+
+  it("returns seeded references for authenticated users", async () => {
+    const user = createTestUser();
+    const { ctx } = createMockContext(user);
+    const caller = appRouter.createCaller(ctx);
+    const seedSpy = vi.spyOn(db, "seedVisionReferenceImages").mockResolvedValueOnce();
+    const listSpy = vi.spyOn(db, "listVisionReferenceImages").mockResolvedValueOnce([
+      {
+        id: 1,
+        slug: "ref-1",
+        title: "标准图：正手挥拍",
+        exerciseType: "forehand",
+        imageUrl: "https://example.com/forehand.jpg",
+        sourcePageUrl: "https://example.com/source",
+        sourceLabel: "Example",
+        author: null,
+        license: null,
+        expectedFocus: ["肩髋转动"],
+        tags: ["forehand"],
+        notes: null,
+        sortOrder: 1,
+        isPublished: 1,
+        createdAt: new Date(),
+        updatedAt: new Date(),
+      },
+    ] as any);
+
+    const result = await caller.vision.library();
+
+    expect(seedSpy).toHaveBeenCalledTimes(1);
+    expect(listSpy).toHaveBeenCalledTimes(1);
+    expect(result).toHaveLength(1);
+  });
+});
+
+describe("vision.runs", () => {
+  it("limits regular users to their own vision test runs", async () => {
+    const user = createTestUser({ id: 7, role: "user" });
+    const { ctx } = createMockContext(user);
+    const caller = appRouter.createCaller(ctx);
+    const listSpy = vi.spyOn(db, "listVisionTestRuns").mockResolvedValueOnce([]);
+
+    await caller.vision.runs({ limit: 20 });
+
+    expect(listSpy).toHaveBeenCalledWith(7, 20);
+  });
+
+  it("allows admin users to view all vision test runs", async () => {
+    const admin = createTestUser({ id: 9, role: "admin", name: "H1" });
+    const { ctx } = createMockContext(admin);
+    const caller = appRouter.createCaller(ctx);
+    const listSpy = vi.spyOn(db, "listVisionTestRuns").mockResolvedValueOnce([]);
+
+    await caller.vision.runs({ limit: 30 });
+
+    expect(listSpy).toHaveBeenCalledWith(undefined, 30);
+  });
+});
+
+describe("vision.seedLibrary", () => {
+  it("rejects non-admin users", async () => {
+    const user = createTestUser({ role: "user" });
+    const { ctx } = createMockContext(user);
+    const caller = appRouter.createCaller(ctx);
+
+    await expect(caller.vision.seedLibrary()).rejects.toThrow();
+  });
+});
--- a/server/routers.ts
+++ b/server/routers.ts
@@ -1,9 +1,10 @@
 import { COOKIE_NAME, ONE_YEAR_MS } from "@shared/const";
 import { getSessionCookieOptions } from "./_core/cookies";
 import { systemRouter } from "./_core/systemRouter";
-import { publicProcedure, protectedProcedure, router } from "./_core/trpc";
+import { adminProcedure, publicProcedure, protectedProcedure, router } from "./_core/trpc";
 import { z } from "zod";
 import { sdk } from "./_core/sdk";
+import { ENV } from "./_core/env";
 import { storagePut } from "./storage";
 import * as db from "./db";
 import { nanoid } from "nanoid";
@@ -271,6 +272,111 @@ export const appRouter = router({
      }),
  }),

+  vision: router({
+    library: protectedProcedure.query(async () => {
+      await db.seedVisionReferenceImages();
+      return db.listVisionReferenceImages();
+    }),
+
+    seedLibrary: adminProcedure.mutation(async () => {
+      await db.seedVisionReferenceImages();
+      const images = await db.listVisionReferenceImages();
+      return { count: images.length };
+    }),
+
+    runs: protectedProcedure
+      .input(z.object({ limit: z.number().min(1).max(100).default(50) }).optional())
+      .query(async ({ ctx, input }) => {
+        const limit = input?.limit ?? 50;
+        return db.listVisionTestRuns(ctx.user.role === "admin" ? undefined : ctx.user.id, limit);
+      }),
+
+    runReference: protectedProcedure
+      .input(z.object({
+        referenceImageId: z.number(),
+        exerciseType: z.string().optional(),
+      }))
+      .mutation(async ({ ctx, input }) => {
+        const reference = await db.getVisionReferenceImageById(input.referenceImageId);
+        if (!reference || reference.isPublished !== 1) {
+          throw new Error("Reference image not found");
+        }
+
+        const task = await enqueueTask({
+          userId: ctx.user.id,
+          type: "pose_correction_multimodal",
+          title: `${reference.title} 视觉测试`,
+          message: "视觉标准图测试已加入后台队列",
+          payload: {
+            poseMetrics: {
+              referenceSource: "vision_reference_library",
+              expectedFocus: reference.expectedFocus,
+              sourcePageUrl: reference.sourcePageUrl,
+            },
+            exerciseType: input.exerciseType || reference.exerciseType,
+            detectedIssues: [],
+            imageUrls: [reference.imageUrl],
+          },
+        });
+
+        const runId = await db.createVisionTestRun({
+          taskId: task.taskId,
+          userId: ctx.user.id,
+          referenceImageId: reference.id,
+          title: `${reference.title} 视觉测试`,
+          exerciseType: input.exerciseType || reference.exerciseType,
+          imageUrl: reference.imageUrl,
+          status: "queued",
+          visionStatus: "pending",
+          configuredModel: ENV.llmVisionModel || null,
+          expectedFocus: reference.expectedFocus,
+        });
+
+        return { taskId: task.taskId, runId };
+      }),
+
+    runAll: protectedProcedure.mutation(async ({ ctx }) => {
+      const references = await db.listVisionReferenceImages();
+      const queued: Array<{ taskId: string; referenceImageId: number }> = [];
+
+      for (const reference of references) {
+        const task = await enqueueTask({
+          userId: ctx.user.id,
+          type: "pose_correction_multimodal",
+          title: `${reference.title} 视觉测试`,
+          message: "视觉标准图测试已加入后台队列",
+          payload: {
+            poseMetrics: {
+              referenceSource: "vision_reference_library",
+              expectedFocus: reference.expectedFocus,
+              sourcePageUrl: reference.sourcePageUrl,
+            },
+            exerciseType: reference.exerciseType,
+            detectedIssues: [],
+            imageUrls: [reference.imageUrl],
+          },
+        });
+
+        await db.createVisionTestRun({
+          taskId: task.taskId,
+          userId: ctx.user.id,
+          referenceImageId: reference.id,
+          title: `${reference.title} 视觉测试`,
+          exerciseType: reference.exerciseType,
+          imageUrl: reference.imageUrl,
+          status: "queued",
+          visionStatus: "pending",
+          configuredModel: ENV.llmVisionModel || null,
+          expectedFocus: reference.expectedFocus,
+        });
+
+        queued.push({ taskId: task.taskId, referenceImageId: reference.id });
+      }
+
+      return { count: queued.length, queued };
+    }),
+  }),
+
  task: router({
    list: protectedProcedure
      .input(z.object({ limit: z.number().min(1).max(50).default(20) }).optional())
--- a/server/taskWorker.ts
+++ b/server/taskWorker.ts
@@ -66,6 +66,33 @@ async function invokeStructured<T>(params: StructuredParams<T>) {
  throw lastError instanceof Error ? lastError : new Error("Failed to parse structured LLM response");
 }

+function contentToPlainText(content: Message["content"]) {
+  if (typeof content === "string") {
+    return content;
+  }
+
+  const parts = Array.isArray(content) ? content : [content];
+
+  return parts
+    .map((part) => {
+      if (typeof part === "string") {
+        return part;
+      }
+      if (part.type === "text") {
+        return part.text;
+      }
+      if (part.type === "image_url") {
+        return `[image] ${part.image_url.url}`;
+      }
+      if (part.type === "file_url") {
+        return `[file] ${part.file_url.url}`;
+      }
+      return "";
+    })
+    .filter(Boolean)
+    .join("\n");
+}
+
 function parseDataUrl(input: string) {
  const match = input.match(/^data:(.+?);base64,(.+)$/);
  if (!match) {
@@ -296,7 +323,7 @@ async function createTextCorrectionResult(payload: {

  return {
    kind: "analysis_corrections" as const,
-    corrections: response.choices[0]?.message?.content || "暂无建议",
+    corrections: contentToPlainText(response.choices[0]?.message?.content || "暂无建议"),
  };
 }

@@ -347,16 +374,26 @@ async function runMultimodalCorrectionTask(task: NonNullable<TaskRow>) {
      },
    });

-    return {
+    const result = {
      kind: "pose_correction_multimodal" as const,
      imageUrls: payload.imageUrls,
      report,
      corrections: renderMultimodalCorrectionMarkdown(report as Parameters<typeof renderMultimodalCorrectionMarkdown>[0]),
      visionStatus: "ok" as const,
    };
+
+    await db.completeVisionTestRun(task.id, {
+      visionStatus: "ok",
+      summary: (report as { summary?: string }).summary ?? null,
+      corrections: result.corrections,
+      report,
+      warning: null,
+    });
+
+    return result;
  } catch (error) {
    const fallback = await createTextCorrectionResult(payload);
-    return {
+    const result = {
      kind: "pose_correction_multimodal" as const,
      imageUrls: payload.imageUrls,
      report: null,
@@ -364,6 +401,16 @@ async function runMultimodalCorrectionTask(task: NonNullable<TaskRow>) {
      visionStatus: "fallback" as const,
      warning: error instanceof Error ? error.message : "Vision model unavailable",
    };
+
+    await db.completeVisionTestRun(task.id, {
+      visionStatus: "fallback",
+      summary: null,
+      corrections: result.corrections,
+      report: null,
+      warning: result.warning,
+    });
+
+    return result;
  }
 }

--- a/server/worker.ts
+++ b/server/worker.ts
@@ -25,6 +25,7 @@ async function workOnce() {
  } catch (error) {
    const message = error instanceof Error ? error.message : "Unknown background task error";
    await db.failBackgroundTask(task.id, message);
+    await db.failVisionTestRun(task.id, message);
    console.error(`[worker] task ${task.id} failed:`, error);
  }