feat: async task pipeline for media and llm workflows
这个提交包含在:
@@ -11,6 +11,7 @@ const parseBoolean = (value: string | undefined, fallback: boolean) => {
|
||||
|
||||
export const ENV = {
|
||||
appId: process.env.VITE_APP_ID ?? "",
|
||||
appPublicBaseUrl: process.env.APP_PUBLIC_BASE_URL ?? "",
|
||||
cookieSecret: process.env.JWT_SECRET ?? "",
|
||||
databaseUrl: process.env.DATABASE_URL ?? "",
|
||||
oAuthServerUrl: process.env.OAUTH_SERVER_URL ?? "",
|
||||
@@ -27,7 +28,22 @@ export const ENV = {
|
||||
llmApiKey:
|
||||
process.env.LLM_API_KEY ?? process.env.BUILT_IN_FORGE_API_KEY ?? "",
|
||||
llmModel: process.env.LLM_MODEL ?? "gemini-2.5-flash",
|
||||
llmVisionApiUrl:
|
||||
process.env.LLM_VISION_API_URL ??
|
||||
process.env.LLM_API_URL ??
|
||||
(process.env.BUILT_IN_FORGE_API_URL
|
||||
? `${process.env.BUILT_IN_FORGE_API_URL.replace(/\/$/, "")}/v1/chat/completions`
|
||||
: ""),
|
||||
llmVisionApiKey:
|
||||
process.env.LLM_VISION_API_KEY ??
|
||||
process.env.LLM_API_KEY ??
|
||||
process.env.BUILT_IN_FORGE_API_KEY ??
|
||||
"",
|
||||
llmVisionModel: process.env.LLM_VISION_MODEL ?? process.env.LLM_MODEL ?? "gemini-2.5-flash",
|
||||
llmMaxTokens: parseInteger(process.env.LLM_MAX_TOKENS, 32768),
|
||||
llmEnableThinking: parseBoolean(process.env.LLM_ENABLE_THINKING, false),
|
||||
llmThinkingBudget: parseInteger(process.env.LLM_THINKING_BUDGET, 128),
|
||||
mediaServiceUrl: process.env.MEDIA_SERVICE_URL ?? "",
|
||||
backgroundTaskPollMs: parseInteger(process.env.BACKGROUND_TASK_POLL_MS, 3000),
|
||||
backgroundTaskStaleMs: parseInteger(process.env.BACKGROUND_TASK_STALE_MS, 300000),
|
||||
};
|
||||
|
||||
@@ -68,6 +68,29 @@ describe("invokeLLM", () => {
|
||||
expect(JSON.parse(request.body)).not.toHaveProperty("thinking");
|
||||
});
|
||||
|
||||
it("allows overriding the model per request", async () => {
|
||||
process.env.LLM_API_URL = "https://one.hao.work/v1/chat/completions";
|
||||
process.env.LLM_API_KEY = "test-key";
|
||||
process.env.LLM_MODEL = "qwen3.5-plus";
|
||||
|
||||
const fetchMock = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => mockSuccessResponse,
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const { invokeLLM } = await import("./llm");
|
||||
await invokeLLM({
|
||||
model: "qwen3-vl-235b-a22b",
|
||||
messages: [{ role: "user", content: "describe image" }],
|
||||
});
|
||||
|
||||
const [, request] = fetchMock.mock.calls[0] as [string, { body: string }];
|
||||
expect(JSON.parse(request.body)).toMatchObject({
|
||||
model: "qwen3-vl-235b-a22b",
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back to legacy forge variables when LLM_* values are absent", async () => {
|
||||
delete process.env.LLM_API_URL;
|
||||
delete process.env.LLM_API_KEY;
|
||||
|
||||
@@ -57,6 +57,9 @@ export type ToolChoice =
|
||||
|
||||
export type InvokeParams = {
|
||||
messages: Message[];
|
||||
model?: string;
|
||||
apiUrl?: string;
|
||||
apiKey?: string;
|
||||
tools?: Tool[];
|
||||
toolChoice?: ToolChoice;
|
||||
tool_choice?: ToolChoice;
|
||||
@@ -209,13 +212,15 @@ const normalizeToolChoice = (
|
||||
return toolChoice;
|
||||
};
|
||||
|
||||
const resolveApiUrl = () =>
|
||||
ENV.llmApiUrl && ENV.llmApiUrl.trim().length > 0
|
||||
const resolveApiUrl = (apiUrl?: string) =>
|
||||
apiUrl && apiUrl.trim().length > 0
|
||||
? apiUrl
|
||||
: ENV.llmApiUrl && ENV.llmApiUrl.trim().length > 0
|
||||
? ENV.llmApiUrl
|
||||
: "https://forge.manus.im/v1/chat/completions";
|
||||
|
||||
const assertApiKey = () => {
|
||||
if (!ENV.llmApiKey) {
|
||||
const assertApiKey = (apiKey?: string) => {
|
||||
if (!(apiKey || ENV.llmApiKey)) {
|
||||
throw new Error("LLM_API_KEY is not configured");
|
||||
}
|
||||
};
|
||||
@@ -266,10 +271,13 @@ const normalizeResponseFormat = ({
|
||||
};
|
||||
|
||||
export async function invokeLLM(params: InvokeParams): Promise<InvokeResult> {
|
||||
assertApiKey();
|
||||
assertApiKey(params.apiKey);
|
||||
|
||||
const {
|
||||
messages,
|
||||
model,
|
||||
apiUrl,
|
||||
apiKey,
|
||||
tools,
|
||||
toolChoice,
|
||||
tool_choice,
|
||||
@@ -280,7 +288,7 @@ export async function invokeLLM(params: InvokeParams): Promise<InvokeResult> {
|
||||
} = params;
|
||||
|
||||
const payload: Record<string, unknown> = {
|
||||
model: ENV.llmModel,
|
||||
model: model || ENV.llmModel,
|
||||
messages: messages.map(normalizeMessage),
|
||||
};
|
||||
|
||||
@@ -315,11 +323,11 @@ export async function invokeLLM(params: InvokeParams): Promise<InvokeResult> {
|
||||
payload.response_format = normalizedResponseFormat;
|
||||
}
|
||||
|
||||
const response = await fetch(resolveApiUrl(), {
|
||||
const response = await fetch(resolveApiUrl(apiUrl), {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
authorization: `Bearer ${ENV.llmApiKey}`,
|
||||
authorization: `Bearer ${apiKey || ENV.llmApiKey}`,
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
|
||||
@@ -6,7 +6,7 @@ export function serveStatic(app: Express) {
|
||||
const distPath =
|
||||
process.env.NODE_ENV === "development"
|
||||
? path.resolve(import.meta.dirname, "../..", "dist", "public")
|
||||
: path.resolve(import.meta.dirname, "public");
|
||||
: path.resolve(import.meta.dirname, "..", "public");
|
||||
if (!fs.existsSync(distPath)) {
|
||||
console.error(
|
||||
`Could not find the build directory: ${distPath}, make sure to build the client first`
|
||||
|
||||
168
server/db.ts
168
server/db.ts
@@ -1,4 +1,4 @@
|
||||
import { eq, desc, and, sql } from "drizzle-orm";
|
||||
import { eq, desc, and, asc, lte, sql } from "drizzle-orm";
|
||||
import { drizzle } from "drizzle-orm/mysql2";
|
||||
import {
|
||||
InsertUser, users,
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
tutorialProgress, InsertTutorialProgress,
|
||||
trainingReminders, InsertTrainingReminder,
|
||||
notificationLog, InsertNotificationLog,
|
||||
backgroundTasks, InsertBackgroundTask,
|
||||
} from "../drizzle/schema";
|
||||
import { ENV } from './_core/env';
|
||||
|
||||
@@ -179,6 +180,15 @@ export async function getVideoById(videoId: number) {
|
||||
return result.length > 0 ? result[0] : undefined;
|
||||
}
|
||||
|
||||
export async function getVideoByFileKey(userId: number, fileKey: string) {
|
||||
const db = await getDb();
|
||||
if (!db) return undefined;
|
||||
const result = await db.select().from(trainingVideos)
|
||||
.where(and(eq(trainingVideos.userId, userId), eq(trainingVideos.fileKey, fileKey)))
|
||||
.limit(1);
|
||||
return result.length > 0 ? result[0] : undefined;
|
||||
}
|
||||
|
||||
export async function updateVideoStatus(videoId: number, status: "pending" | "analyzing" | "completed" | "failed") {
|
||||
const db = await getDb();
|
||||
if (!db) return;
|
||||
@@ -660,6 +670,162 @@ export async function getUnreadNotificationCount(userId: number) {
|
||||
return result[0]?.count || 0;
|
||||
}
|
||||
|
||||
// ===== BACKGROUND TASK OPERATIONS =====
|
||||
|
||||
export async function createBackgroundTask(task: InsertBackgroundTask) {
|
||||
const db = await getDb();
|
||||
if (!db) throw new Error("Database not available");
|
||||
await db.insert(backgroundTasks).values(task);
|
||||
return task.id;
|
||||
}
|
||||
|
||||
export async function listUserBackgroundTasks(userId: number, limit = 20) {
|
||||
const db = await getDb();
|
||||
if (!db) return [];
|
||||
return db.select().from(backgroundTasks)
|
||||
.where(eq(backgroundTasks.userId, userId))
|
||||
.orderBy(desc(backgroundTasks.createdAt))
|
||||
.limit(limit);
|
||||
}
|
||||
|
||||
export async function getBackgroundTaskById(taskId: string) {
|
||||
const db = await getDb();
|
||||
if (!db) return undefined;
|
||||
const result = await db.select().from(backgroundTasks)
|
||||
.where(eq(backgroundTasks.id, taskId))
|
||||
.limit(1);
|
||||
return result[0];
|
||||
}
|
||||
|
||||
export async function getUserBackgroundTaskById(userId: number, taskId: string) {
|
||||
const db = await getDb();
|
||||
if (!db) return undefined;
|
||||
const result = await db.select().from(backgroundTasks)
|
||||
.where(and(eq(backgroundTasks.id, taskId), eq(backgroundTasks.userId, userId)))
|
||||
.limit(1);
|
||||
return result[0];
|
||||
}
|
||||
|
||||
export async function claimNextBackgroundTask(workerId: string) {
|
||||
const db = await getDb();
|
||||
if (!db) return null;
|
||||
|
||||
const now = new Date();
|
||||
const [nextTask] = await db.select().from(backgroundTasks)
|
||||
.where(and(eq(backgroundTasks.status, "queued"), lte(backgroundTasks.runAfter, now)))
|
||||
.orderBy(asc(backgroundTasks.runAfter), asc(backgroundTasks.createdAt))
|
||||
.limit(1);
|
||||
|
||||
if (!nextTask) {
|
||||
return null;
|
||||
}
|
||||
|
||||
await db.update(backgroundTasks).set({
|
||||
status: "running",
|
||||
workerId,
|
||||
attempts: sql`${backgroundTasks.attempts} + 1`,
|
||||
lockedAt: now,
|
||||
startedAt: now,
|
||||
updatedAt: now,
|
||||
}).where(eq(backgroundTasks.id, nextTask.id));
|
||||
|
||||
return getBackgroundTaskById(nextTask.id);
|
||||
}
|
||||
|
||||
export async function heartbeatBackgroundTask(taskId: string, workerId: string) {
|
||||
const db = await getDb();
|
||||
if (!db) return;
|
||||
await db.update(backgroundTasks).set({
|
||||
workerId,
|
||||
lockedAt: new Date(),
|
||||
}).where(eq(backgroundTasks.id, taskId));
|
||||
}
|
||||
|
||||
export async function updateBackgroundTask(taskId: string, data: Partial<InsertBackgroundTask>) {
|
||||
const db = await getDb();
|
||||
if (!db) return;
|
||||
await db.update(backgroundTasks).set(data).where(eq(backgroundTasks.id, taskId));
|
||||
}
|
||||
|
||||
export async function completeBackgroundTask(taskId: string, result: unknown, message?: string) {
|
||||
const db = await getDb();
|
||||
if (!db) return;
|
||||
await db.update(backgroundTasks).set({
|
||||
status: "succeeded",
|
||||
progress: 100,
|
||||
message: message ?? "已完成",
|
||||
result,
|
||||
error: null,
|
||||
workerId: null,
|
||||
lockedAt: null,
|
||||
completedAt: new Date(),
|
||||
}).where(eq(backgroundTasks.id, taskId));
|
||||
}
|
||||
|
||||
export async function failBackgroundTask(taskId: string, error: string) {
|
||||
const db = await getDb();
|
||||
if (!db) return;
|
||||
await db.update(backgroundTasks).set({
|
||||
status: "failed",
|
||||
error,
|
||||
workerId: null,
|
||||
lockedAt: null,
|
||||
completedAt: new Date(),
|
||||
}).where(eq(backgroundTasks.id, taskId));
|
||||
}
|
||||
|
||||
export async function rescheduleBackgroundTask(taskId: string, params: {
|
||||
progress?: number;
|
||||
message?: string;
|
||||
error?: string | null;
|
||||
delayMs?: number;
|
||||
}) {
|
||||
const db = await getDb();
|
||||
if (!db) return;
|
||||
await db.update(backgroundTasks).set({
|
||||
status: "queued",
|
||||
progress: params.progress,
|
||||
message: params.message,
|
||||
error: params.error ?? null,
|
||||
workerId: null,
|
||||
lockedAt: null,
|
||||
runAfter: new Date(Date.now() + (params.delayMs ?? 0)),
|
||||
}).where(eq(backgroundTasks.id, taskId));
|
||||
}
|
||||
|
||||
export async function retryBackgroundTask(userId: number, taskId: string) {
|
||||
const db = await getDb();
|
||||
if (!db) throw new Error("Database not available");
|
||||
const task = await getUserBackgroundTaskById(userId, taskId);
|
||||
if (!task) {
|
||||
throw new Error("Task not found");
|
||||
}
|
||||
await db.update(backgroundTasks).set({
|
||||
status: "queued",
|
||||
progress: 0,
|
||||
message: "任务已重新排队",
|
||||
error: null,
|
||||
result: null,
|
||||
workerId: null,
|
||||
lockedAt: null,
|
||||
completedAt: null,
|
||||
runAfter: new Date(),
|
||||
}).where(eq(backgroundTasks.id, taskId));
|
||||
return getBackgroundTaskById(taskId);
|
||||
}
|
||||
|
||||
export async function requeueStaleBackgroundTasks(staleBefore: Date) {
|
||||
const db = await getDb();
|
||||
if (!db) return;
|
||||
await db.update(backgroundTasks).set({
|
||||
status: "queued",
|
||||
message: "检测到任务中断,已重新排队",
|
||||
workerId: null,
|
||||
lockedAt: null,
|
||||
runAfter: new Date(),
|
||||
}).where(and(eq(backgroundTasks.status, "running"), lte(backgroundTasks.lockedAt, staleBefore)));
|
||||
}
|
||||
|
||||
// ===== STATS HELPERS =====
|
||||
|
||||
export async function getUserStats(userId: number) {
|
||||
|
||||
34
server/mediaService.ts
普通文件
34
server/mediaService.ts
普通文件
@@ -0,0 +1,34 @@
|
||||
import { ENV } from "./_core/env";
|
||||
|
||||
export type RemoteMediaSession = {
|
||||
id: string;
|
||||
userId: string;
|
||||
title: string;
|
||||
archiveStatus: "idle" | "queued" | "processing" | "completed" | "failed";
|
||||
playback: {
|
||||
webmUrl?: string;
|
||||
mp4Url?: string;
|
||||
webmSize?: number;
|
||||
mp4Size?: number;
|
||||
ready: boolean;
|
||||
previewUrl?: string;
|
||||
};
|
||||
lastError?: string;
|
||||
};
|
||||
|
||||
function getMediaBaseUrl() {
|
||||
if (!ENV.mediaServiceUrl) {
|
||||
throw new Error("MEDIA_SERVICE_URL is not configured");
|
||||
}
|
||||
return ENV.mediaServiceUrl.replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
export async function getRemoteMediaSession(sessionId: string) {
|
||||
const response = await fetch(`${getMediaBaseUrl()}/sessions/${sessionId}`);
|
||||
if (!response.ok) {
|
||||
const message = await response.text().catch(() => response.statusText);
|
||||
throw new Error(`Media service request failed (${response.status}): ${message}`);
|
||||
}
|
||||
const payload = await response.json() as { session: RemoteMediaSession };
|
||||
return payload.session;
|
||||
}
|
||||
255
server/prompts.ts
普通文件
255
server/prompts.ts
普通文件
@@ -0,0 +1,255 @@
|
||||
type RecentScore = {
|
||||
score: number | null;
|
||||
issues: unknown;
|
||||
exerciseType: string | null;
|
||||
shotCount: number | null;
|
||||
strokeConsistency: number | null;
|
||||
footworkScore: number | null;
|
||||
};
|
||||
|
||||
type RecentAnalysis = {
|
||||
score: number | null;
|
||||
issues: unknown;
|
||||
corrections: unknown;
|
||||
shotCount: number | null;
|
||||
strokeConsistency: number | null;
|
||||
footworkScore: number | null;
|
||||
fluidityScore: number | null;
|
||||
};
|
||||
|
||||
function skillLevelLabel(skillLevel: "beginner" | "intermediate" | "advanced") {
|
||||
switch (skillLevel) {
|
||||
case "intermediate":
|
||||
return "中级";
|
||||
case "advanced":
|
||||
return "高级";
|
||||
default:
|
||||
return "初级";
|
||||
}
|
||||
}
|
||||
|
||||
export function buildTrainingPlanPrompt(input: {
|
||||
skillLevel: "beginner" | "intermediate" | "advanced";
|
||||
durationDays: number;
|
||||
focusAreas?: string[];
|
||||
recentScores: RecentScore[];
|
||||
}) {
|
||||
return [
|
||||
`你是一位专业网球教练。请为一位${skillLevelLabel(input.skillLevel)}水平的网球学员生成 ${input.durationDays} 天训练计划。`,
|
||||
"训练条件与要求:",
|
||||
"- 训练以个人可执行为主,可使用球拍、弹力带、标志盘、墙面等常见器材。",
|
||||
"- 每天训练 30-60 分钟,结构要清晰:热身、专项、脚步、力量/稳定、放松。",
|
||||
"- 输出内容要适合直接执行,不写空话,不写营销语,不写额外说明。",
|
||||
input.focusAreas?.length ? `- 重点关注:${input.focusAreas.join("、")}` : "- 如未指定重点,请自动平衡技术、脚步和体能。",
|
||||
input.recentScores.length > 0
|
||||
? `- 用户最近分析摘要:${JSON.stringify(input.recentScores)}`
|
||||
: "- 暂无历史分析数据,请基于该水平的常见薄弱项设计。",
|
||||
"每个训练项都要给出目标、动作描述、组次/次数、关键提示,避免重复堆砌。",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function buildAdjustedTrainingPlanPrompt(input: {
|
||||
currentExercises: unknown;
|
||||
recentAnalyses: RecentAnalysis[];
|
||||
}) {
|
||||
return [
|
||||
"你是一位专业网球教练,需要根据最近训练分析结果调整现有训练计划。",
|
||||
`当前计划:${JSON.stringify(input.currentExercises)}`,
|
||||
`最近分析结果:${JSON.stringify(input.recentAnalyses)}`,
|
||||
"请优先修复最近最频繁、最影响击球质量的问题。",
|
||||
"要求:",
|
||||
"- 保留原计划中仍然有效的训练项,不要全部推倒重来。",
|
||||
"- 增加动作纠正、脚步节奏、稳定性和专项力量训练。",
|
||||
"- adjustmentNotes 需要说明为什么这样调整,以及下一阶段重点。",
|
||||
"- 输出仅返回结构化 JSON。",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function buildTextCorrectionPrompt(input: {
|
||||
exerciseType: string;
|
||||
poseMetrics: unknown;
|
||||
detectedIssues: unknown;
|
||||
}) {
|
||||
return [
|
||||
"你是一位网球技术教练与动作纠正分析师。",
|
||||
`动作类型:${input.exerciseType}`,
|
||||
`姿态指标:${JSON.stringify(input.poseMetrics)}`,
|
||||
`已检测问题:${JSON.stringify(input.detectedIssues)}`,
|
||||
"请用中文输出专业、直接、可执行的纠正建议,使用 Markdown。",
|
||||
"内容结构必须包括:",
|
||||
"1. 动作概览",
|
||||
"2. 最高优先级的 3 个修正点",
|
||||
"3. 每个修正点对应的练习方法、感受提示、完成标准",
|
||||
"4. 下一次拍摄或训练时的注意事项",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export const multimodalCorrectionSchema = {
|
||||
type: "object",
|
||||
properties: {
|
||||
summary: { type: "string" },
|
||||
overallScore: { type: "number" },
|
||||
confidence: { type: "number" },
|
||||
phaseFindings: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
properties: {
|
||||
phase: { type: "string" },
|
||||
score: { type: "number" },
|
||||
observation: { type: "string" },
|
||||
impact: { type: "string" },
|
||||
},
|
||||
required: ["phase", "score", "observation", "impact"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
bodyPartFindings: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
properties: {
|
||||
bodyPart: { type: "string" },
|
||||
issue: { type: "string" },
|
||||
recommendation: { type: "string" },
|
||||
},
|
||||
required: ["bodyPart", "issue", "recommendation"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
priorityFixes: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
properties: {
|
||||
title: { type: "string" },
|
||||
why: { type: "string" },
|
||||
howToPractice: { type: "string" },
|
||||
successMetric: { type: "string" },
|
||||
},
|
||||
required: ["title", "why", "howToPractice", "successMetric"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
drills: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
properties: {
|
||||
name: { type: "string" },
|
||||
purpose: { type: "string" },
|
||||
durationMinutes: { type: "number" },
|
||||
steps: {
|
||||
type: "array",
|
||||
items: { type: "string" },
|
||||
},
|
||||
coachingCues: {
|
||||
type: "array",
|
||||
items: { type: "string" },
|
||||
},
|
||||
},
|
||||
required: ["name", "purpose", "durationMinutes", "steps", "coachingCues"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
safetyRisks: {
|
||||
type: "array",
|
||||
items: { type: "string" },
|
||||
},
|
||||
nextSessionFocus: {
|
||||
type: "array",
|
||||
items: { type: "string" },
|
||||
},
|
||||
recommendedCaptureTips: {
|
||||
type: "array",
|
||||
items: { type: "string" },
|
||||
},
|
||||
},
|
||||
required: [
|
||||
"summary",
|
||||
"overallScore",
|
||||
"confidence",
|
||||
"phaseFindings",
|
||||
"bodyPartFindings",
|
||||
"priorityFixes",
|
||||
"drills",
|
||||
"safetyRisks",
|
||||
"nextSessionFocus",
|
||||
"recommendedCaptureTips",
|
||||
],
|
||||
additionalProperties: false,
|
||||
};
|
||||
|
||||
export function buildMultimodalCorrectionPrompt(input: {
|
||||
exerciseType: string;
|
||||
poseMetrics: unknown;
|
||||
detectedIssues: unknown;
|
||||
imageCount: number;
|
||||
}) {
|
||||
return [
|
||||
"你是一位专业网球技术教练,正在审阅学员的动作截图。",
|
||||
`动作类型:${input.exerciseType}`,
|
||||
`结构化姿态指标:${JSON.stringify(input.poseMetrics)}`,
|
||||
`已有问题标签:${JSON.stringify(input.detectedIssues)}`,
|
||||
`本次共提供 ${input.imageCount} 张关键帧图片。`,
|
||||
"请严格依据图片和结构化指标交叉判断,不要编造看不到的动作细节。",
|
||||
"分析要求:",
|
||||
"- 识别准备、引拍、击球/发力、收拍几个阶段的质量。",
|
||||
"- 指出躯干、肩髋、击球臂、非持拍手、重心和脚步的主要问题。",
|
||||
"- priorityFixes 只保留最重要、最值得优先修正的项目。",
|
||||
"- drills 要足够具体,适合下一次训练直接执行。",
|
||||
"- recommendedCaptureTips 说明下次如何补拍,以便提高判断准确度。",
|
||||
"输出仅返回 JSON,不要附加解释。",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function renderMultimodalCorrectionMarkdown(report: {
|
||||
summary: string;
|
||||
overallScore: number;
|
||||
confidence: number;
|
||||
priorityFixes: Array<{ title: string; why: string; howToPractice: string; successMetric: string }>;
|
||||
drills: Array<{ name: string; purpose: string; durationMinutes: number; coachingCues: string[] }>;
|
||||
safetyRisks: string[];
|
||||
nextSessionFocus: string[];
|
||||
recommendedCaptureTips: string[];
|
||||
}) {
|
||||
const priorityFixes = report.priorityFixes
|
||||
.map((item, index) => [
|
||||
`${index + 1}. ${item.title}`,
|
||||
`- 原因:${item.why}`,
|
||||
`- 练习:${item.howToPractice}`,
|
||||
`- 达标:${item.successMetric}`,
|
||||
].join("\n"))
|
||||
.join("\n");
|
||||
|
||||
const drills = report.drills
|
||||
.map((item) => [
|
||||
`- ${item.name}(${item.durationMinutes} 分钟)`,
|
||||
` 目的:${item.purpose}`,
|
||||
` 口令:${item.coachingCues.join(";")}`,
|
||||
].join("\n"))
|
||||
.join("\n");
|
||||
|
||||
return [
|
||||
`## 动作概览`,
|
||||
report.summary,
|
||||
"",
|
||||
`- 综合评分:${Math.round(report.overallScore)}/100`,
|
||||
`- 置信度:${Math.round(report.confidence)}%`,
|
||||
"",
|
||||
"## 优先修正",
|
||||
priorityFixes || "- 暂无",
|
||||
"",
|
||||
"## 推荐练习",
|
||||
drills || "- 暂无",
|
||||
"",
|
||||
"## 风险提醒",
|
||||
report.safetyRisks.length > 0 ? report.safetyRisks.map(item => `- ${item}`).join("\n") : "- 暂无明显风险",
|
||||
"",
|
||||
"## 下次训练重点",
|
||||
report.nextSessionFocus.length > 0 ? report.nextSessionFocus.map(item => `- ${item}`).join("\n") : "- 保持当前节奏",
|
||||
"",
|
||||
"## 下次拍摄建议",
|
||||
report.recommendedCaptureTips.length > 0 ? report.recommendedCaptureTips.map(item => `- ${item}`).join("\n") : "- 保持当前拍摄方式",
|
||||
].join("\n");
|
||||
}
|
||||
22
server/publicUrl.ts
普通文件
22
server/publicUrl.ts
普通文件
@@ -0,0 +1,22 @@
|
||||
import { ENV } from "./_core/env";
|
||||
|
||||
function hasProtocol(value: string) {
|
||||
return /^[a-z][a-z0-9+.-]*:\/\//i.test(value);
|
||||
}
|
||||
|
||||
export function toPublicUrl(pathOrUrl: string) {
|
||||
const value = pathOrUrl.trim();
|
||||
if (!value) {
|
||||
throw new Error("Public URL value is empty");
|
||||
}
|
||||
|
||||
if (hasProtocol(value)) {
|
||||
return value;
|
||||
}
|
||||
|
||||
if (!ENV.appPublicBaseUrl) {
|
||||
throw new Error("APP_PUBLIC_BASE_URL is required for externally accessible asset URLs");
|
||||
}
|
||||
|
||||
return new URL(value.startsWith("/") ? value : `/${value}`, ENV.appPublicBaseUrl).toString();
|
||||
}
|
||||
@@ -4,53 +4,34 @@ import { systemRouter } from "./_core/systemRouter";
|
||||
import { publicProcedure, protectedProcedure, router } from "./_core/trpc";
|
||||
import { z } from "zod";
|
||||
import { sdk } from "./_core/sdk";
|
||||
import { invokeLLM } from "./_core/llm";
|
||||
import { storagePut } from "./storage";
|
||||
import * as db from "./db";
|
||||
import { nanoid } from "nanoid";
|
||||
import {
|
||||
normalizeAdjustedPlanResponse,
|
||||
normalizeTrainingPlanResponse,
|
||||
} from "./trainingPlan";
|
||||
import { getRemoteMediaSession } from "./mediaService";
|
||||
import { prepareCorrectionImageUrls } from "./taskWorker";
|
||||
import { toPublicUrl } from "./publicUrl";
|
||||
|
||||
async function invokeStructuredPlan<T>(params: {
|
||||
baseMessages: Array<{ role: "system" | "user"; content: string }>;
|
||||
responseFormat: {
|
||||
type: "json_schema";
|
||||
json_schema: {
|
||||
name: string;
|
||||
strict: true;
|
||||
schema: Record<string, unknown>;
|
||||
};
|
||||
};
|
||||
parse: (content: unknown) => T;
|
||||
async function enqueueTask(params: {
|
||||
userId: number;
|
||||
type: "media_finalize" | "training_plan_generate" | "training_plan_adjust" | "analysis_corrections" | "pose_correction_multimodal";
|
||||
title: string;
|
||||
payload: Record<string, unknown>;
|
||||
message: string;
|
||||
}) {
|
||||
let lastError: unknown;
|
||||
const taskId = nanoid();
|
||||
await db.createBackgroundTask({
|
||||
id: taskId,
|
||||
userId: params.userId,
|
||||
type: params.type,
|
||||
title: params.title,
|
||||
message: params.message,
|
||||
payload: params.payload,
|
||||
progress: 0,
|
||||
maxAttempts: params.type === "media_finalize" ? 90 : 3,
|
||||
});
|
||||
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
const retryHint =
|
||||
attempt === 0 || !(lastError instanceof Error)
|
||||
? []
|
||||
: [{
|
||||
role: "user" as const,
|
||||
content:
|
||||
`上一次输出无法被系统解析,错误是:${lastError.message}。` +
|
||||
"请只返回一个合法、完整、可解析的 JSON 对象,不要包含额外说明、注释或 Markdown 代码块。",
|
||||
}];
|
||||
|
||||
const response = await invokeLLM({
|
||||
messages: [...params.baseMessages, ...retryHint],
|
||||
response_format: params.responseFormat,
|
||||
});
|
||||
|
||||
try {
|
||||
return params.parse(response.choices[0]?.message?.content);
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError instanceof Error ? lastError : new Error("Failed to parse structured LLM response");
|
||||
const task = await db.getBackgroundTaskById(taskId);
|
||||
return { taskId, task };
|
||||
}
|
||||
|
||||
export const appRouter = router({
|
||||
@@ -104,86 +85,13 @@ export const appRouter = router({
|
||||
focusAreas: z.array(z.string()).optional(),
|
||||
}))
|
||||
.mutation(async ({ ctx, input }) => {
|
||||
const user = ctx.user;
|
||||
// Get user's recent analyses for personalization
|
||||
const analyses = await db.getUserAnalyses(user.id);
|
||||
const recentScores = analyses.slice(0, 5).map(a => ({
|
||||
score: a.overallScore,
|
||||
issues: a.detectedIssues,
|
||||
exerciseType: a.exerciseType,
|
||||
shotCount: a.shotCount,
|
||||
strokeConsistency: a.strokeConsistency,
|
||||
footworkScore: a.footworkScore,
|
||||
}));
|
||||
|
||||
const prompt = `你是一位网球教练。请为一位${
|
||||
input.skillLevel === "beginner" ? "初级" : input.skillLevel === "intermediate" ? "中级" : "高级"
|
||||
}水平的网球学员生成一个${input.durationDays}天的训练计划。
|
||||
|
||||
要求:
|
||||
- 只需要球拍,不需要球场和球网
|
||||
- 包含影子挥拍、墙壁练习、脚步移动、体能训练等
|
||||
- 每天训练30-60分钟
|
||||
${input.focusAreas?.length ? `- 重点关注: ${input.focusAreas.join(", ")}` : ""}
|
||||
${recentScores.length > 0 ? `- 用户最近的分析数据: ${JSON.stringify(recentScores)}` : ""}
|
||||
|
||||
请返回JSON格式,包含每天的训练内容。`;
|
||||
|
||||
const parsed = await invokeStructuredPlan({
|
||||
baseMessages: [
|
||||
{ role: "system", content: "你是网球训练计划生成器。返回严格的JSON格式。" },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
responseFormat: {
|
||||
type: "json_schema",
|
||||
json_schema: {
|
||||
name: "training_plan",
|
||||
strict: true,
|
||||
schema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
title: { type: "string", description: "训练计划标题" },
|
||||
exercises: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
properties: {
|
||||
day: { type: "number" },
|
||||
name: { type: "string" },
|
||||
category: { type: "string" },
|
||||
duration: { type: "number", description: "分钟" },
|
||||
description: { type: "string" },
|
||||
tips: { type: "string" },
|
||||
sets: { type: "number" },
|
||||
reps: { type: "number" },
|
||||
},
|
||||
required: ["day", "name", "category", "duration", "description", "tips", "sets", "reps"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
required: ["title", "exercises"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
parse: (content) => normalizeTrainingPlanResponse({
|
||||
content,
|
||||
fallbackTitle: `${input.durationDays}天训练计划`,
|
||||
}),
|
||||
return enqueueTask({
|
||||
userId: ctx.user.id,
|
||||
type: "training_plan_generate",
|
||||
title: `${input.durationDays}天训练计划生成`,
|
||||
message: "训练计划已加入后台队列",
|
||||
payload: input,
|
||||
});
|
||||
|
||||
const planId = await db.createTrainingPlan({
|
||||
userId: user.id,
|
||||
title: parsed.title,
|
||||
skillLevel: input.skillLevel,
|
||||
durationDays: input.durationDays,
|
||||
exercises: parsed.exercises,
|
||||
isActive: 1,
|
||||
version: 1,
|
||||
});
|
||||
|
||||
return { planId, plan: parsed };
|
||||
}),
|
||||
|
||||
list: protectedProcedure.query(async ({ ctx }) => {
|
||||
@@ -197,78 +105,15 @@ ${recentScores.length > 0 ? `- 用户最近的分析数据: ${JSON.stringify(rec
|
||||
adjust: protectedProcedure
|
||||
.input(z.object({ planId: z.number() }))
|
||||
.mutation(async ({ ctx, input }) => {
|
||||
const analyses = await db.getUserAnalyses(ctx.user.id);
|
||||
const recentAnalyses = analyses.slice(0, 5);
|
||||
const currentPlan = (await db.getUserTrainingPlans(ctx.user.id)).find(p => p.id === input.planId);
|
||||
if (!currentPlan) throw new Error("Plan not found");
|
||||
|
||||
const prompt = `基于以下用户的姿势分析结果,调整训练计划:
|
||||
|
||||
当前计划: ${JSON.stringify(currentPlan.exercises)}
|
||||
最近分析结果: ${JSON.stringify(recentAnalyses.map(a => ({
|
||||
score: a.overallScore,
|
||||
issues: a.detectedIssues,
|
||||
corrections: a.corrections,
|
||||
shotCount: a.shotCount,
|
||||
strokeConsistency: a.strokeConsistency,
|
||||
footworkScore: a.footworkScore,
|
||||
fluidityScore: a.fluidityScore,
|
||||
})))}
|
||||
|
||||
请根据分析结果调整训练计划,增加针对薄弱环节的训练,返回与原计划相同格式的JSON。`;
|
||||
|
||||
const parsed = await invokeStructuredPlan({
|
||||
baseMessages: [
|
||||
{ role: "system", content: "你是网球训练计划调整器。返回严格的JSON格式。" },
|
||||
{ role: "user", content: prompt },
|
||||
],
|
||||
responseFormat: {
|
||||
type: "json_schema",
|
||||
json_schema: {
|
||||
name: "adjusted_plan",
|
||||
strict: true,
|
||||
schema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
title: { type: "string" },
|
||||
adjustmentNotes: { type: "string", description: "调整说明" },
|
||||
exercises: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
properties: {
|
||||
day: { type: "number" },
|
||||
name: { type: "string" },
|
||||
category: { type: "string" },
|
||||
duration: { type: "number" },
|
||||
description: { type: "string" },
|
||||
tips: { type: "string" },
|
||||
sets: { type: "number" },
|
||||
reps: { type: "number" },
|
||||
},
|
||||
required: ["day", "name", "category", "duration", "description", "tips", "sets", "reps"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
required: ["title", "adjustmentNotes", "exercises"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
parse: (content) => normalizeAdjustedPlanResponse({
|
||||
content,
|
||||
fallbackTitle: currentPlan.title,
|
||||
}),
|
||||
return enqueueTask({
|
||||
userId: ctx.user.id,
|
||||
type: "training_plan_adjust",
|
||||
title: `${currentPlan.title} 调整`,
|
||||
message: "训练计划调整任务已加入后台队列",
|
||||
payload: input,
|
||||
});
|
||||
|
||||
await db.updateTrainingPlan(input.planId, {
|
||||
exercises: parsed.exercises,
|
||||
adjustmentNotes: parsed.adjustmentNotes,
|
||||
version: (currentPlan.version || 1) + 1,
|
||||
});
|
||||
|
||||
return { success: true, adjustmentNotes: parsed.adjustmentNotes };
|
||||
}),
|
||||
}),
|
||||
|
||||
@@ -287,19 +132,20 @@ ${recentScores.length > 0 ? `- 用户最近的分析数据: ${JSON.stringify(rec
|
||||
const fileKey = `videos/${ctx.user.id}/${nanoid()}.${input.format}`;
|
||||
const contentType = input.format === "webm" ? "video/webm" : "video/mp4";
|
||||
const { url } = await storagePut(fileKey, fileBuffer, contentType);
|
||||
const publicUrl = toPublicUrl(url);
|
||||
|
||||
const videoId = await db.createVideo({
|
||||
userId: ctx.user.id,
|
||||
title: input.title,
|
||||
fileKey,
|
||||
url,
|
||||
url: publicUrl,
|
||||
format: input.format,
|
||||
fileSize: input.fileSize,
|
||||
exerciseType: input.exerciseType || null,
|
||||
analysisStatus: "pending",
|
||||
});
|
||||
|
||||
return { videoId, url };
|
||||
return { videoId, url: publicUrl };
|
||||
}),
|
||||
|
||||
registerExternal: protectedProcedure
|
||||
@@ -313,11 +159,12 @@ ${recentScores.length > 0 ? `- 用户最近的分析数据: ${JSON.stringify(rec
|
||||
exerciseType: z.string().optional(),
|
||||
}))
|
||||
.mutation(async ({ ctx, input }) => {
|
||||
const publicUrl = toPublicUrl(input.url);
|
||||
const videoId = await db.createVideo({
|
||||
userId: ctx.user.id,
|
||||
title: input.title,
|
||||
fileKey: input.fileKey,
|
||||
url: input.url,
|
||||
url: publicUrl,
|
||||
format: input.format,
|
||||
fileSize: input.fileSize ?? null,
|
||||
duration: input.duration ?? null,
|
||||
@@ -325,7 +172,7 @@ ${recentScores.length > 0 ? `- 用户最近的分析数据: ${JSON.stringify(rec
|
||||
analysisStatus: "completed",
|
||||
});
|
||||
|
||||
return { videoId, url: input.url };
|
||||
return { videoId, url: publicUrl };
|
||||
}),
|
||||
|
||||
list: protectedProcedure.query(async ({ ctx }) => {
|
||||
@@ -399,32 +246,70 @@ ${recentScores.length > 0 ? `- 用户最近的分析数据: ${JSON.stringify(rec
|
||||
poseMetrics: z.any(),
|
||||
exerciseType: z.string(),
|
||||
detectedIssues: z.any(),
|
||||
imageUrls: z.array(z.string()).optional(),
|
||||
imageDataUrls: z.array(z.string()).max(4).optional(),
|
||||
}))
|
||||
.mutation(async ({ input }) => {
|
||||
const response = await invokeLLM({
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: "你是一位网球动作分析员。根据MediaPipe姿势分析数据,给出具体的姿势矫正建议。用中文回答。",
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: `分析以下网球动作数据并给出矫正建议:
|
||||
动作类型: ${input.exerciseType}
|
||||
姿势指标: ${JSON.stringify(input.poseMetrics)}
|
||||
检测到的问题: ${JSON.stringify(input.detectedIssues)}
|
||||
|
||||
请给出:
|
||||
1. 每个问题的具体矫正方法
|
||||
2. 推荐的练习动作
|
||||
3. 需要注意的关键点`,
|
||||
},
|
||||
],
|
||||
.mutation(async ({ ctx, input }) => {
|
||||
const imageUrls = await prepareCorrectionImageUrls({
|
||||
userId: ctx.user.id,
|
||||
imageUrls: input.imageUrls,
|
||||
imageDataUrls: input.imageDataUrls,
|
||||
});
|
||||
|
||||
return {
|
||||
corrections: response.choices[0]?.message?.content || "暂无建议",
|
||||
};
|
||||
return enqueueTask({
|
||||
userId: ctx.user.id,
|
||||
type: imageUrls.length > 0 ? "pose_correction_multimodal" : "analysis_corrections",
|
||||
title: `${input.exerciseType} 动作纠正`,
|
||||
message: imageUrls.length > 0 ? "多模态动作纠正任务已加入后台队列" : "动作纠正任务已加入后台队列",
|
||||
payload: {
|
||||
poseMetrics: input.poseMetrics,
|
||||
exerciseType: input.exerciseType,
|
||||
detectedIssues: input.detectedIssues,
|
||||
imageUrls,
|
||||
},
|
||||
});
|
||||
}),
|
||||
}),
|
||||
|
||||
task: router({
|
||||
list: protectedProcedure
|
||||
.input(z.object({ limit: z.number().min(1).max(50).default(20) }).optional())
|
||||
.query(async ({ ctx, input }) => {
|
||||
return db.listUserBackgroundTasks(ctx.user.id, input?.limit ?? 20);
|
||||
}),
|
||||
|
||||
get: protectedProcedure
|
||||
.input(z.object({ taskId: z.string().min(1) }))
|
||||
.query(async ({ ctx, input }) => {
|
||||
return db.getUserBackgroundTaskById(ctx.user.id, input.taskId);
|
||||
}),
|
||||
|
||||
retry: protectedProcedure
|
||||
.input(z.object({ taskId: z.string().min(1) }))
|
||||
.mutation(async ({ ctx, input }) => {
|
||||
const task = await db.retryBackgroundTask(ctx.user.id, input.taskId);
|
||||
return { task };
|
||||
}),
|
||||
|
||||
createMediaFinalize: protectedProcedure
|
||||
.input(z.object({
|
||||
sessionId: z.string().min(1),
|
||||
title: z.string().min(1).max(256),
|
||||
exerciseType: z.string().optional(),
|
||||
}))
|
||||
.mutation(async ({ ctx, input }) => {
|
||||
const session = await getRemoteMediaSession(input.sessionId);
|
||||
if (session.userId !== String(ctx.user.id)) {
|
||||
throw new Error("Media session not found");
|
||||
}
|
||||
|
||||
return enqueueTask({
|
||||
userId: ctx.user.id,
|
||||
type: "media_finalize",
|
||||
title: `${input.title} 归档`,
|
||||
message: "录制文件归档任务已加入后台队列",
|
||||
payload: input,
|
||||
});
|
||||
}),
|
||||
}),
|
||||
|
||||
|
||||
@@ -37,4 +37,16 @@ describe("storage fallback", () => {
|
||||
url: "/uploads/videos/test/sample.webm",
|
||||
});
|
||||
});
|
||||
|
||||
it("builds externally accessible URLs for local assets", async () => {
|
||||
process.env.APP_PUBLIC_BASE_URL = "https://te.hao.work/";
|
||||
const { toExternalAssetUrl } = await import("./storage");
|
||||
|
||||
expect(toExternalAssetUrl("/uploads/videos/test/sample.webm")).toBe(
|
||||
"https://te.hao.work/uploads/videos/test/sample.webm"
|
||||
);
|
||||
expect(toExternalAssetUrl("https://cdn.example.com/demo.jpg")).toBe(
|
||||
"https://cdn.example.com/demo.jpg"
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { ENV } from './_core/env';
|
||||
import { toPublicUrl } from "./publicUrl";
|
||||
|
||||
type StorageConfig = { baseUrl: string; apiKey: string };
|
||||
|
||||
@@ -141,3 +142,7 @@ export async function storageGet(relKey: string): Promise<{ key: string; url: st
|
||||
url: await buildDownloadUrl(baseUrl, key, apiKey),
|
||||
};
|
||||
}
|
||||
|
||||
export function toExternalAssetUrl(pathOrUrl: string) {
|
||||
return toPublicUrl(pathOrUrl);
|
||||
}
|
||||
|
||||
470
server/taskWorker.ts
普通文件
470
server/taskWorker.ts
普通文件
@@ -0,0 +1,470 @@
|
||||
import { nanoid } from "nanoid";
|
||||
import { ENV } from "./_core/env";
|
||||
import { invokeLLM, type Message } from "./_core/llm";
|
||||
import * as db from "./db";
|
||||
import { getRemoteMediaSession } from "./mediaService";
|
||||
import {
|
||||
buildAdjustedTrainingPlanPrompt,
|
||||
buildMultimodalCorrectionPrompt,
|
||||
buildTextCorrectionPrompt,
|
||||
buildTrainingPlanPrompt,
|
||||
multimodalCorrectionSchema,
|
||||
renderMultimodalCorrectionMarkdown,
|
||||
} from "./prompts";
|
||||
import { toPublicUrl } from "./publicUrl";
|
||||
import { storagePut } from "./storage";
|
||||
import {
|
||||
normalizeAdjustedPlanResponse,
|
||||
normalizeTrainingPlanResponse,
|
||||
} from "./trainingPlan";
|
||||
|
||||
type TaskRow = Awaited<ReturnType<typeof db.getBackgroundTaskById>>;
|
||||
|
||||
type StructuredParams<T> = {
|
||||
model?: string;
|
||||
baseMessages: Array<{ role: "system" | "user"; content: string | Message["content"] }>;
|
||||
responseFormat: {
|
||||
type: "json_schema";
|
||||
json_schema: {
|
||||
name: string;
|
||||
strict: true;
|
||||
schema: Record<string, unknown>;
|
||||
};
|
||||
};
|
||||
parse: (content: unknown) => T;
|
||||
};
|
||||
|
||||
async function invokeStructured<T>(params: StructuredParams<T>) {
|
||||
let lastError: unknown;
|
||||
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
const retryHint =
|
||||
attempt === 0 || !(lastError instanceof Error)
|
||||
? []
|
||||
: [{
|
||||
role: "user" as const,
|
||||
content:
|
||||
`上一次输出无法被系统解析,错误是:${lastError.message}。` +
|
||||
"请只返回合法完整的 JSON 对象,不要附加 Markdown 或说明。",
|
||||
}];
|
||||
|
||||
const response = await invokeLLM({
|
||||
apiUrl: params.model === ENV.llmVisionModel ? ENV.llmVisionApiUrl : undefined,
|
||||
apiKey: params.model === ENV.llmVisionModel ? ENV.llmVisionApiKey : undefined,
|
||||
model: params.model,
|
||||
messages: [...params.baseMessages, ...retryHint],
|
||||
response_format: params.responseFormat,
|
||||
});
|
||||
|
||||
try {
|
||||
return params.parse(response.choices[0]?.message?.content);
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError instanceof Error ? lastError : new Error("Failed to parse structured LLM response");
|
||||
}
|
||||
|
||||
function parseDataUrl(input: string) {
|
||||
const match = input.match(/^data:(.+?);base64,(.+)$/);
|
||||
if (!match) {
|
||||
throw new Error("Invalid image data URL");
|
||||
}
|
||||
return {
|
||||
contentType: match[1],
|
||||
buffer: Buffer.from(match[2], "base64"),
|
||||
};
|
||||
}
|
||||
|
||||
async function persistInlineImages(userId: number, imageDataUrls: string[]) {
|
||||
const persistedUrls: string[] = [];
|
||||
for (let index = 0; index < imageDataUrls.length; index++) {
|
||||
const { contentType, buffer } = parseDataUrl(imageDataUrls[index]);
|
||||
const extension = contentType.includes("png") ? "png" : "jpg";
|
||||
const key = `analysis-images/${userId}/${nanoid()}.${extension}`;
|
||||
const uploaded = await storagePut(key, buffer, contentType);
|
||||
persistedUrls.push(toPublicUrl(uploaded.url));
|
||||
}
|
||||
return persistedUrls;
|
||||
}
|
||||
|
||||
export async function prepareCorrectionImageUrls(input: {
|
||||
userId: number;
|
||||
imageUrls?: string[];
|
||||
imageDataUrls?: string[];
|
||||
}) {
|
||||
const directUrls = (input.imageUrls ?? []).map((item) => toPublicUrl(item));
|
||||
const uploadedUrls = input.imageDataUrls?.length
|
||||
? await persistInlineImages(input.userId, input.imageDataUrls)
|
||||
: [];
|
||||
return [...directUrls, ...uploadedUrls];
|
||||
}
|
||||
|
||||
async function runTrainingPlanGenerateTask(task: NonNullable<TaskRow>) {
|
||||
const payload = task.payload as {
|
||||
skillLevel: "beginner" | "intermediate" | "advanced";
|
||||
durationDays: number;
|
||||
focusAreas?: string[];
|
||||
};
|
||||
const analyses = await db.getUserAnalyses(task.userId);
|
||||
const recentScores = analyses.slice(0, 5).map((analysis) => ({
|
||||
score: analysis.overallScore ?? null,
|
||||
issues: analysis.detectedIssues,
|
||||
exerciseType: analysis.exerciseType ?? null,
|
||||
shotCount: analysis.shotCount ?? null,
|
||||
strokeConsistency: analysis.strokeConsistency ?? null,
|
||||
footworkScore: analysis.footworkScore ?? null,
|
||||
}));
|
||||
|
||||
const parsed = await invokeStructured({
|
||||
baseMessages: [
|
||||
{ role: "system", content: "你是网球训练计划生成器。返回严格的 JSON 格式。" },
|
||||
{
|
||||
role: "user",
|
||||
content: buildTrainingPlanPrompt({
|
||||
...payload,
|
||||
recentScores,
|
||||
}),
|
||||
},
|
||||
],
|
||||
responseFormat: {
|
||||
type: "json_schema",
|
||||
json_schema: {
|
||||
name: "training_plan",
|
||||
strict: true,
|
||||
schema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
title: { type: "string" },
|
||||
exercises: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
properties: {
|
||||
day: { type: "number" },
|
||||
name: { type: "string" },
|
||||
category: { type: "string" },
|
||||
duration: { type: "number" },
|
||||
description: { type: "string" },
|
||||
tips: { type: "string" },
|
||||
sets: { type: "number" },
|
||||
reps: { type: "number" },
|
||||
},
|
||||
required: ["day", "name", "category", "duration", "description", "tips", "sets", "reps"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
required: ["title", "exercises"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
parse: (content) => normalizeTrainingPlanResponse({
|
||||
content,
|
||||
fallbackTitle: `${payload.durationDays}天训练计划`,
|
||||
}),
|
||||
});
|
||||
|
||||
const planId = await db.createTrainingPlan({
|
||||
userId: task.userId,
|
||||
title: parsed.title,
|
||||
skillLevel: payload.skillLevel,
|
||||
durationDays: payload.durationDays,
|
||||
exercises: parsed.exercises,
|
||||
isActive: 1,
|
||||
version: 1,
|
||||
});
|
||||
|
||||
return {
|
||||
kind: "training_plan_generate" as const,
|
||||
planId,
|
||||
plan: parsed,
|
||||
};
|
||||
}
|
||||
|
||||
async function runTrainingPlanAdjustTask(task: NonNullable<TaskRow>) {
|
||||
const payload = task.payload as { planId: number };
|
||||
const analyses = await db.getUserAnalyses(task.userId);
|
||||
const recentAnalyses = analyses.slice(0, 5);
|
||||
const currentPlan = (await db.getUserTrainingPlans(task.userId)).find((plan) => plan.id === payload.planId);
|
||||
|
||||
if (!currentPlan) {
|
||||
throw new Error("Plan not found");
|
||||
}
|
||||
|
||||
const parsed = await invokeStructured({
|
||||
baseMessages: [
|
||||
{ role: "system", content: "你是网球训练计划调整器。返回严格的 JSON 格式。" },
|
||||
{
|
||||
role: "user",
|
||||
content: buildAdjustedTrainingPlanPrompt({
|
||||
currentExercises: currentPlan.exercises,
|
||||
recentAnalyses: recentAnalyses.map((analysis) => ({
|
||||
score: analysis.overallScore ?? null,
|
||||
issues: analysis.detectedIssues,
|
||||
corrections: analysis.corrections,
|
||||
shotCount: analysis.shotCount ?? null,
|
||||
strokeConsistency: analysis.strokeConsistency ?? null,
|
||||
footworkScore: analysis.footworkScore ?? null,
|
||||
fluidityScore: analysis.fluidityScore ?? null,
|
||||
})),
|
||||
}),
|
||||
},
|
||||
],
|
||||
responseFormat: {
|
||||
type: "json_schema",
|
||||
json_schema: {
|
||||
name: "adjusted_plan",
|
||||
strict: true,
|
||||
schema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
title: { type: "string" },
|
||||
adjustmentNotes: { type: "string" },
|
||||
exercises: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
properties: {
|
||||
day: { type: "number" },
|
||||
name: { type: "string" },
|
||||
category: { type: "string" },
|
||||
duration: { type: "number" },
|
||||
description: { type: "string" },
|
||||
tips: { type: "string" },
|
||||
sets: { type: "number" },
|
||||
reps: { type: "number" },
|
||||
},
|
||||
required: ["day", "name", "category", "duration", "description", "tips", "sets", "reps"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
required: ["title", "adjustmentNotes", "exercises"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
parse: (content) => normalizeAdjustedPlanResponse({
|
||||
content,
|
||||
fallbackTitle: currentPlan.title,
|
||||
}),
|
||||
});
|
||||
|
||||
await db.updateTrainingPlan(payload.planId, {
|
||||
exercises: parsed.exercises,
|
||||
adjustmentNotes: parsed.adjustmentNotes,
|
||||
version: (currentPlan.version || 1) + 1,
|
||||
});
|
||||
|
||||
return {
|
||||
kind: "training_plan_adjust" as const,
|
||||
planId: payload.planId,
|
||||
plan: parsed,
|
||||
adjustmentNotes: parsed.adjustmentNotes,
|
||||
};
|
||||
}
|
||||
|
||||
async function runTextCorrectionTask(task: NonNullable<TaskRow>) {
|
||||
const payload = task.payload as {
|
||||
exerciseType: string;
|
||||
poseMetrics: unknown;
|
||||
detectedIssues: unknown;
|
||||
};
|
||||
return createTextCorrectionResult(payload);
|
||||
}
|
||||
|
||||
async function createTextCorrectionResult(payload: {
|
||||
exerciseType: string;
|
||||
poseMetrics: unknown;
|
||||
detectedIssues: unknown;
|
||||
}) {
|
||||
const response = await invokeLLM({
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: "你是一位专业网球技术教练。输出中文 Markdown,内容具体、克制、可执行。",
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: buildTextCorrectionPrompt(payload),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
return {
|
||||
kind: "analysis_corrections" as const,
|
||||
corrections: response.choices[0]?.message?.content || "暂无建议",
|
||||
};
|
||||
}
|
||||
|
||||
async function runMultimodalCorrectionTask(task: NonNullable<TaskRow>) {
|
||||
const payload = task.payload as {
|
||||
exerciseType: string;
|
||||
poseMetrics: unknown;
|
||||
detectedIssues: unknown;
|
||||
imageUrls: string[];
|
||||
};
|
||||
try {
|
||||
const report = await invokeStructured({
|
||||
model: ENV.llmVisionModel,
|
||||
baseMessages: [
|
||||
{ role: "system", content: "你是专业网球教练。请基于图片和结构化姿态指标输出严格 JSON。" },
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: buildMultimodalCorrectionPrompt({
|
||||
exerciseType: payload.exerciseType,
|
||||
poseMetrics: payload.poseMetrics,
|
||||
detectedIssues: payload.detectedIssues,
|
||||
imageCount: payload.imageUrls.length,
|
||||
}) },
|
||||
...payload.imageUrls.map((url) => ({
|
||||
type: "image_url" as const,
|
||||
image_url: {
|
||||
url,
|
||||
detail: "high" as const,
|
||||
},
|
||||
})),
|
||||
],
|
||||
},
|
||||
],
|
||||
responseFormat: {
|
||||
type: "json_schema",
|
||||
json_schema: {
|
||||
name: "pose_correction_multimodal",
|
||||
strict: true,
|
||||
schema: multimodalCorrectionSchema,
|
||||
},
|
||||
},
|
||||
parse: (content) => {
|
||||
if (typeof content === "string") {
|
||||
return JSON.parse(content);
|
||||
}
|
||||
return content as Record<string, unknown>;
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
kind: "pose_correction_multimodal" as const,
|
||||
imageUrls: payload.imageUrls,
|
||||
report,
|
||||
corrections: renderMultimodalCorrectionMarkdown(report as Parameters<typeof renderMultimodalCorrectionMarkdown>[0]),
|
||||
visionStatus: "ok" as const,
|
||||
};
|
||||
} catch (error) {
|
||||
const fallback = await createTextCorrectionResult(payload);
|
||||
return {
|
||||
kind: "pose_correction_multimodal" as const,
|
||||
imageUrls: payload.imageUrls,
|
||||
report: null,
|
||||
corrections: fallback.corrections,
|
||||
visionStatus: "fallback" as const,
|
||||
warning: error instanceof Error ? error.message : "Vision model unavailable",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function runMediaFinalizeTask(task: NonNullable<TaskRow>) {
|
||||
const payload = task.payload as {
|
||||
sessionId: string;
|
||||
title: string;
|
||||
exerciseType?: string;
|
||||
};
|
||||
const session = await getRemoteMediaSession(payload.sessionId);
|
||||
|
||||
if (session.userId !== String(task.userId)) {
|
||||
throw new Error("Media session does not belong to the task user");
|
||||
}
|
||||
|
||||
if (session.archiveStatus === "queued") {
|
||||
await db.rescheduleBackgroundTask(task.id, {
|
||||
progress: 45,
|
||||
message: "录制文件已入队,等待归档",
|
||||
delayMs: 4_000,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
if (session.archiveStatus === "processing") {
|
||||
await db.rescheduleBackgroundTask(task.id, {
|
||||
progress: 78,
|
||||
message: "录制文件正在整理与转码",
|
||||
delayMs: 4_000,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
if (session.archiveStatus === "failed") {
|
||||
throw new Error(session.lastError || "Media archive failed");
|
||||
}
|
||||
|
||||
if (!session.playback.ready) {
|
||||
await db.rescheduleBackgroundTask(task.id, {
|
||||
progress: 70,
|
||||
message: "等待回放文件就绪",
|
||||
delayMs: 4_000,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
const preferredUrl = session.playback.mp4Url || session.playback.webmUrl;
|
||||
const format = session.playback.mp4Url ? "mp4" : "webm";
|
||||
if (!preferredUrl) {
|
||||
throw new Error("Media session did not expose a playback URL");
|
||||
}
|
||||
|
||||
const fileKey = `media/sessions/${session.id}/recording.${format}`;
|
||||
const existing = await db.getVideoByFileKey(task.userId, fileKey);
|
||||
if (existing) {
|
||||
return {
|
||||
kind: "media_finalize" as const,
|
||||
sessionId: session.id,
|
||||
videoId: existing.id,
|
||||
url: existing.url,
|
||||
fileKey,
|
||||
format,
|
||||
};
|
||||
}
|
||||
|
||||
const publicUrl = toPublicUrl(preferredUrl);
|
||||
const videoId = await db.createVideo({
|
||||
userId: task.userId,
|
||||
title: payload.title || session.title,
|
||||
fileKey,
|
||||
url: publicUrl,
|
||||
format,
|
||||
fileSize: format === "mp4" ? (session.playback.mp4Size ?? null) : (session.playback.webmSize ?? null),
|
||||
duration: null,
|
||||
exerciseType: payload.exerciseType || "recording",
|
||||
analysisStatus: "completed",
|
||||
});
|
||||
|
||||
return {
|
||||
kind: "media_finalize" as const,
|
||||
sessionId: session.id,
|
||||
videoId,
|
||||
url: publicUrl,
|
||||
fileKey,
|
||||
format,
|
||||
};
|
||||
}
|
||||
|
||||
export async function processBackgroundTask(task: NonNullable<TaskRow>) {
|
||||
switch (task.type) {
|
||||
case "training_plan_generate":
|
||||
return runTrainingPlanGenerateTask(task);
|
||||
case "training_plan_adjust":
|
||||
return runTrainingPlanAdjustTask(task);
|
||||
case "analysis_corrections":
|
||||
return runTextCorrectionTask(task);
|
||||
case "pose_correction_multimodal":
|
||||
return runMultimodalCorrectionTask(task);
|
||||
case "media_finalize":
|
||||
return runMediaFinalizeTask(task);
|
||||
default:
|
||||
throw new Error(`Unsupported task type: ${String(task.type)}`);
|
||||
}
|
||||
}
|
||||
47
server/worker.ts
普通文件
47
server/worker.ts
普通文件
@@ -0,0 +1,47 @@
|
||||
import "dotenv/config";
|
||||
import { ENV } from "./_core/env";
|
||||
import * as db from "./db";
|
||||
import { processBackgroundTask } from "./taskWorker";
|
||||
|
||||
const workerId = `app-worker-${process.pid}`;
|
||||
|
||||
function sleep(ms: number) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function workOnce() {
|
||||
await db.requeueStaleBackgroundTasks(new Date(Date.now() - ENV.backgroundTaskStaleMs));
|
||||
|
||||
const task = await db.claimNextBackgroundTask(workerId);
|
||||
if (!task) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await processBackgroundTask(task);
|
||||
if (result !== null) {
|
||||
await db.completeBackgroundTask(task.id, result, "任务执行完成");
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Unknown background task error";
|
||||
await db.failBackgroundTask(task.id, message);
|
||||
console.error(`[worker] task ${task.id} failed:`, error);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log(`[worker] ${workerId} started`);
|
||||
for (;;) {
|
||||
const hasWorked = await workOnce();
|
||||
if (!hasWorked) {
|
||||
await sleep(ENV.backgroundTaskPollMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error("[worker] fatal error", error);
|
||||
process.exit(1);
|
||||
});
|
||||
在新工单中引用
屏蔽一个用户