feat: async task pipeline for media and llm workflows

这个提交包含在:
cryptocommuniums-afk
2026-03-15 00:12:26 +08:00
父节点 1cc863e60e
当前提交 20e183d2da
修改 36 个文件,包含 1961 行新增339 行删除

查看文件

@@ -11,6 +11,7 @@ const parseBoolean = (value: string | undefined, fallback: boolean) => {
export const ENV = {
appId: process.env.VITE_APP_ID ?? "",
appPublicBaseUrl: process.env.APP_PUBLIC_BASE_URL ?? "",
cookieSecret: process.env.JWT_SECRET ?? "",
databaseUrl: process.env.DATABASE_URL ?? "",
oAuthServerUrl: process.env.OAUTH_SERVER_URL ?? "",
@@ -27,7 +28,22 @@ export const ENV = {
llmApiKey:
process.env.LLM_API_KEY ?? process.env.BUILT_IN_FORGE_API_KEY ?? "",
llmModel: process.env.LLM_MODEL ?? "gemini-2.5-flash",
llmVisionApiUrl:
process.env.LLM_VISION_API_URL ??
process.env.LLM_API_URL ??
(process.env.BUILT_IN_FORGE_API_URL
? `${process.env.BUILT_IN_FORGE_API_URL.replace(/\/$/, "")}/v1/chat/completions`
: ""),
llmVisionApiKey:
process.env.LLM_VISION_API_KEY ??
process.env.LLM_API_KEY ??
process.env.BUILT_IN_FORGE_API_KEY ??
"",
llmVisionModel: process.env.LLM_VISION_MODEL ?? process.env.LLM_MODEL ?? "gemini-2.5-flash",
llmMaxTokens: parseInteger(process.env.LLM_MAX_TOKENS, 32768),
llmEnableThinking: parseBoolean(process.env.LLM_ENABLE_THINKING, false),
llmThinkingBudget: parseInteger(process.env.LLM_THINKING_BUDGET, 128),
mediaServiceUrl: process.env.MEDIA_SERVICE_URL ?? "",
backgroundTaskPollMs: parseInteger(process.env.BACKGROUND_TASK_POLL_MS, 3000),
backgroundTaskStaleMs: parseInteger(process.env.BACKGROUND_TASK_STALE_MS, 300000),
};

查看文件

@@ -68,6 +68,29 @@ describe("invokeLLM", () => {
expect(JSON.parse(request.body)).not.toHaveProperty("thinking");
});
it("allows overriding the model per request", async () => {
process.env.LLM_API_URL = "https://one.hao.work/v1/chat/completions";
process.env.LLM_API_KEY = "test-key";
process.env.LLM_MODEL = "qwen3.5-plus";
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => mockSuccessResponse,
});
vi.stubGlobal("fetch", fetchMock);
const { invokeLLM } = await import("./llm");
await invokeLLM({
model: "qwen3-vl-235b-a22b",
messages: [{ role: "user", content: "describe image" }],
});
const [, request] = fetchMock.mock.calls[0] as [string, { body: string }];
expect(JSON.parse(request.body)).toMatchObject({
model: "qwen3-vl-235b-a22b",
});
});
it("falls back to legacy forge variables when LLM_* values are absent", async () => {
delete process.env.LLM_API_URL;
delete process.env.LLM_API_KEY;

查看文件

@@ -57,6 +57,9 @@ export type ToolChoice =
export type InvokeParams = {
messages: Message[];
model?: string;
apiUrl?: string;
apiKey?: string;
tools?: Tool[];
toolChoice?: ToolChoice;
tool_choice?: ToolChoice;
@@ -209,13 +212,15 @@ const normalizeToolChoice = (
return toolChoice;
};
const resolveApiUrl = () =>
ENV.llmApiUrl && ENV.llmApiUrl.trim().length > 0
const resolveApiUrl = (apiUrl?: string) =>
apiUrl && apiUrl.trim().length > 0
? apiUrl
: ENV.llmApiUrl && ENV.llmApiUrl.trim().length > 0
? ENV.llmApiUrl
: "https://forge.manus.im/v1/chat/completions";
const assertApiKey = () => {
if (!ENV.llmApiKey) {
const assertApiKey = (apiKey?: string) => {
if (!(apiKey || ENV.llmApiKey)) {
throw new Error("LLM_API_KEY is not configured");
}
};
@@ -266,10 +271,13 @@ const normalizeResponseFormat = ({
};
export async function invokeLLM(params: InvokeParams): Promise<InvokeResult> {
assertApiKey();
assertApiKey(params.apiKey);
const {
messages,
model,
apiUrl,
apiKey,
tools,
toolChoice,
tool_choice,
@@ -280,7 +288,7 @@ export async function invokeLLM(params: InvokeParams): Promise<InvokeResult> {
} = params;
const payload: Record<string, unknown> = {
model: ENV.llmModel,
model: model || ENV.llmModel,
messages: messages.map(normalizeMessage),
};
@@ -315,11 +323,11 @@ export async function invokeLLM(params: InvokeParams): Promise<InvokeResult> {
payload.response_format = normalizedResponseFormat;
}
const response = await fetch(resolveApiUrl(), {
const response = await fetch(resolveApiUrl(apiUrl), {
method: "POST",
headers: {
"content-type": "application/json",
authorization: `Bearer ${ENV.llmApiKey}`,
authorization: `Bearer ${apiKey || ENV.llmApiKey}`,
},
body: JSON.stringify(payload),
});

查看文件

@@ -6,7 +6,7 @@ export function serveStatic(app: Express) {
const distPath =
process.env.NODE_ENV === "development"
? path.resolve(import.meta.dirname, "../..", "dist", "public")
: path.resolve(import.meta.dirname, "public");
: path.resolve(import.meta.dirname, "..", "public");
if (!fs.existsSync(distPath)) {
console.error(
`Could not find the build directory: ${distPath}, make sure to build the client first`

查看文件

@@ -1,4 +1,4 @@
import { eq, desc, and, sql } from "drizzle-orm";
import { eq, desc, and, asc, lte, sql } from "drizzle-orm";
import { drizzle } from "drizzle-orm/mysql2";
import {
InsertUser, users,
@@ -14,6 +14,7 @@ import {
tutorialProgress, InsertTutorialProgress,
trainingReminders, InsertTrainingReminder,
notificationLog, InsertNotificationLog,
backgroundTasks, InsertBackgroundTask,
} from "../drizzle/schema";
import { ENV } from './_core/env';
@@ -179,6 +180,15 @@ export async function getVideoById(videoId: number) {
return result.length > 0 ? result[0] : undefined;
}
export async function getVideoByFileKey(userId: number, fileKey: string) {
const db = await getDb();
if (!db) return undefined;
const result = await db.select().from(trainingVideos)
.where(and(eq(trainingVideos.userId, userId), eq(trainingVideos.fileKey, fileKey)))
.limit(1);
return result.length > 0 ? result[0] : undefined;
}
export async function updateVideoStatus(videoId: number, status: "pending" | "analyzing" | "completed" | "failed") {
const db = await getDb();
if (!db) return;
@@ -660,6 +670,162 @@ export async function getUnreadNotificationCount(userId: number) {
return result[0]?.count || 0;
}
// ===== BACKGROUND TASK OPERATIONS =====
export async function createBackgroundTask(task: InsertBackgroundTask) {
const db = await getDb();
if (!db) throw new Error("Database not available");
await db.insert(backgroundTasks).values(task);
return task.id;
}
export async function listUserBackgroundTasks(userId: number, limit = 20) {
const db = await getDb();
if (!db) return [];
return db.select().from(backgroundTasks)
.where(eq(backgroundTasks.userId, userId))
.orderBy(desc(backgroundTasks.createdAt))
.limit(limit);
}
export async function getBackgroundTaskById(taskId: string) {
const db = await getDb();
if (!db) return undefined;
const result = await db.select().from(backgroundTasks)
.where(eq(backgroundTasks.id, taskId))
.limit(1);
return result[0];
}
export async function getUserBackgroundTaskById(userId: number, taskId: string) {
const db = await getDb();
if (!db) return undefined;
const result = await db.select().from(backgroundTasks)
.where(and(eq(backgroundTasks.id, taskId), eq(backgroundTasks.userId, userId)))
.limit(1);
return result[0];
}
export async function claimNextBackgroundTask(workerId: string) {
const db = await getDb();
if (!db) return null;
const now = new Date();
const [nextTask] = await db.select().from(backgroundTasks)
.where(and(eq(backgroundTasks.status, "queued"), lte(backgroundTasks.runAfter, now)))
.orderBy(asc(backgroundTasks.runAfter), asc(backgroundTasks.createdAt))
.limit(1);
if (!nextTask) {
return null;
}
await db.update(backgroundTasks).set({
status: "running",
workerId,
attempts: sql`${backgroundTasks.attempts} + 1`,
lockedAt: now,
startedAt: now,
updatedAt: now,
}).where(eq(backgroundTasks.id, nextTask.id));
return getBackgroundTaskById(nextTask.id);
}
export async function heartbeatBackgroundTask(taskId: string, workerId: string) {
const db = await getDb();
if (!db) return;
await db.update(backgroundTasks).set({
workerId,
lockedAt: new Date(),
}).where(eq(backgroundTasks.id, taskId));
}
export async function updateBackgroundTask(taskId: string, data: Partial<InsertBackgroundTask>) {
const db = await getDb();
if (!db) return;
await db.update(backgroundTasks).set(data).where(eq(backgroundTasks.id, taskId));
}
export async function completeBackgroundTask(taskId: string, result: unknown, message?: string) {
const db = await getDb();
if (!db) return;
await db.update(backgroundTasks).set({
status: "succeeded",
progress: 100,
message: message ?? "已完成",
result,
error: null,
workerId: null,
lockedAt: null,
completedAt: new Date(),
}).where(eq(backgroundTasks.id, taskId));
}
export async function failBackgroundTask(taskId: string, error: string) {
const db = await getDb();
if (!db) return;
await db.update(backgroundTasks).set({
status: "failed",
error,
workerId: null,
lockedAt: null,
completedAt: new Date(),
}).where(eq(backgroundTasks.id, taskId));
}
export async function rescheduleBackgroundTask(taskId: string, params: {
progress?: number;
message?: string;
error?: string | null;
delayMs?: number;
}) {
const db = await getDb();
if (!db) return;
await db.update(backgroundTasks).set({
status: "queued",
progress: params.progress,
message: params.message,
error: params.error ?? null,
workerId: null,
lockedAt: null,
runAfter: new Date(Date.now() + (params.delayMs ?? 0)),
}).where(eq(backgroundTasks.id, taskId));
}
export async function retryBackgroundTask(userId: number, taskId: string) {
const db = await getDb();
if (!db) throw new Error("Database not available");
const task = await getUserBackgroundTaskById(userId, taskId);
if (!task) {
throw new Error("Task not found");
}
await db.update(backgroundTasks).set({
status: "queued",
progress: 0,
message: "任务已重新排队",
error: null,
result: null,
workerId: null,
lockedAt: null,
completedAt: null,
runAfter: new Date(),
}).where(eq(backgroundTasks.id, taskId));
return getBackgroundTaskById(taskId);
}
export async function requeueStaleBackgroundTasks(staleBefore: Date) {
const db = await getDb();
if (!db) return;
await db.update(backgroundTasks).set({
status: "queued",
message: "检测到任务中断,已重新排队",
workerId: null,
lockedAt: null,
runAfter: new Date(),
}).where(and(eq(backgroundTasks.status, "running"), lte(backgroundTasks.lockedAt, staleBefore)));
}
// ===== STATS HELPERS =====
export async function getUserStats(userId: number) {

34
server/mediaService.ts 普通文件
查看文件

@@ -0,0 +1,34 @@
import { ENV } from "./_core/env";
export type RemoteMediaSession = {
id: string;
userId: string;
title: string;
archiveStatus: "idle" | "queued" | "processing" | "completed" | "failed";
playback: {
webmUrl?: string;
mp4Url?: string;
webmSize?: number;
mp4Size?: number;
ready: boolean;
previewUrl?: string;
};
lastError?: string;
};
function getMediaBaseUrl() {
if (!ENV.mediaServiceUrl) {
throw new Error("MEDIA_SERVICE_URL is not configured");
}
return ENV.mediaServiceUrl.replace(/\/+$/, "");
}
export async function getRemoteMediaSession(sessionId: string) {
const response = await fetch(`${getMediaBaseUrl()}/sessions/${sessionId}`);
if (!response.ok) {
const message = await response.text().catch(() => response.statusText);
throw new Error(`Media service request failed (${response.status}): ${message}`);
}
const payload = await response.json() as { session: RemoteMediaSession };
return payload.session;
}

255
server/prompts.ts 普通文件
查看文件

@@ -0,0 +1,255 @@
type RecentScore = {
score: number | null;
issues: unknown;
exerciseType: string | null;
shotCount: number | null;
strokeConsistency: number | null;
footworkScore: number | null;
};
type RecentAnalysis = {
score: number | null;
issues: unknown;
corrections: unknown;
shotCount: number | null;
strokeConsistency: number | null;
footworkScore: number | null;
fluidityScore: number | null;
};
function skillLevelLabel(skillLevel: "beginner" | "intermediate" | "advanced") {
switch (skillLevel) {
case "intermediate":
return "中级";
case "advanced":
return "高级";
default:
return "初级";
}
}
export function buildTrainingPlanPrompt(input: {
skillLevel: "beginner" | "intermediate" | "advanced";
durationDays: number;
focusAreas?: string[];
recentScores: RecentScore[];
}) {
return [
`你是一位专业网球教练。请为一位${skillLevelLabel(input.skillLevel)}水平的网球学员生成 ${input.durationDays} 天训练计划。`,
"训练条件与要求:",
"- 训练以个人可执行为主,可使用球拍、弹力带、标志盘、墙面等常见器材。",
"- 每天训练 30-60 分钟,结构要清晰:热身、专项、脚步、力量/稳定、放松。",
"- 输出内容要适合直接执行,不写空话,不写营销语,不写额外说明。",
input.focusAreas?.length ? `- 重点关注:${input.focusAreas.join("、")}` : "- 如未指定重点,请自动平衡技术、脚步和体能。",
input.recentScores.length > 0
? `- 用户最近分析摘要:${JSON.stringify(input.recentScores)}`
: "- 暂无历史分析数据,请基于该水平的常见薄弱项设计。",
"每个训练项都要给出目标、动作描述、组次/次数、关键提示,避免重复堆砌。",
].join("\n");
}
export function buildAdjustedTrainingPlanPrompt(input: {
currentExercises: unknown;
recentAnalyses: RecentAnalysis[];
}) {
return [
"你是一位专业网球教练,需要根据最近训练分析结果调整现有训练计划。",
`当前计划:${JSON.stringify(input.currentExercises)}`,
`最近分析结果:${JSON.stringify(input.recentAnalyses)}`,
"请优先修复最近最频繁、最影响击球质量的问题。",
"要求:",
"- 保留原计划中仍然有效的训练项,不要全部推倒重来。",
"- 增加动作纠正、脚步节奏、稳定性和专项力量训练。",
"- adjustmentNotes 需要说明为什么这样调整,以及下一阶段重点。",
"- 输出仅返回结构化 JSON。",
].join("\n");
}
export function buildTextCorrectionPrompt(input: {
exerciseType: string;
poseMetrics: unknown;
detectedIssues: unknown;
}) {
return [
"你是一位网球技术教练与动作纠正分析师。",
`动作类型:${input.exerciseType}`,
`姿态指标:${JSON.stringify(input.poseMetrics)}`,
`已检测问题:${JSON.stringify(input.detectedIssues)}`,
"请用中文输出专业、直接、可执行的纠正建议,使用 Markdown。",
"内容结构必须包括:",
"1. 动作概览",
"2. 最高优先级的 3 个修正点",
"3. 每个修正点对应的练习方法、感受提示、完成标准",
"4. 下一次拍摄或训练时的注意事项",
].join("\n");
}
export const multimodalCorrectionSchema = {
type: "object",
properties: {
summary: { type: "string" },
overallScore: { type: "number" },
confidence: { type: "number" },
phaseFindings: {
type: "array",
items: {
type: "object",
properties: {
phase: { type: "string" },
score: { type: "number" },
observation: { type: "string" },
impact: { type: "string" },
},
required: ["phase", "score", "observation", "impact"],
additionalProperties: false,
},
},
bodyPartFindings: {
type: "array",
items: {
type: "object",
properties: {
bodyPart: { type: "string" },
issue: { type: "string" },
recommendation: { type: "string" },
},
required: ["bodyPart", "issue", "recommendation"],
additionalProperties: false,
},
},
priorityFixes: {
type: "array",
items: {
type: "object",
properties: {
title: { type: "string" },
why: { type: "string" },
howToPractice: { type: "string" },
successMetric: { type: "string" },
},
required: ["title", "why", "howToPractice", "successMetric"],
additionalProperties: false,
},
},
drills: {
type: "array",
items: {
type: "object",
properties: {
name: { type: "string" },
purpose: { type: "string" },
durationMinutes: { type: "number" },
steps: {
type: "array",
items: { type: "string" },
},
coachingCues: {
type: "array",
items: { type: "string" },
},
},
required: ["name", "purpose", "durationMinutes", "steps", "coachingCues"],
additionalProperties: false,
},
},
safetyRisks: {
type: "array",
items: { type: "string" },
},
nextSessionFocus: {
type: "array",
items: { type: "string" },
},
recommendedCaptureTips: {
type: "array",
items: { type: "string" },
},
},
required: [
"summary",
"overallScore",
"confidence",
"phaseFindings",
"bodyPartFindings",
"priorityFixes",
"drills",
"safetyRisks",
"nextSessionFocus",
"recommendedCaptureTips",
],
additionalProperties: false,
};
export function buildMultimodalCorrectionPrompt(input: {
exerciseType: string;
poseMetrics: unknown;
detectedIssues: unknown;
imageCount: number;
}) {
return [
"你是一位专业网球技术教练,正在审阅学员的动作截图。",
`动作类型:${input.exerciseType}`,
`结构化姿态指标:${JSON.stringify(input.poseMetrics)}`,
`已有问题标签:${JSON.stringify(input.detectedIssues)}`,
`本次共提供 ${input.imageCount} 张关键帧图片。`,
"请严格依据图片和结构化指标交叉判断,不要编造看不到的动作细节。",
"分析要求:",
"- 识别准备、引拍、击球/发力、收拍几个阶段的质量。",
"- 指出躯干、肩髋、击球臂、非持拍手、重心和脚步的主要问题。",
"- priorityFixes 只保留最重要、最值得优先修正的项目。",
"- drills 要足够具体,适合下一次训练直接执行。",
"- recommendedCaptureTips 说明下次如何补拍,以便提高判断准确度。",
"输出仅返回 JSON,不要附加解释。",
].join("\n");
}
export function renderMultimodalCorrectionMarkdown(report: {
summary: string;
overallScore: number;
confidence: number;
priorityFixes: Array<{ title: string; why: string; howToPractice: string; successMetric: string }>;
drills: Array<{ name: string; purpose: string; durationMinutes: number; coachingCues: string[] }>;
safetyRisks: string[];
nextSessionFocus: string[];
recommendedCaptureTips: string[];
}) {
const priorityFixes = report.priorityFixes
.map((item, index) => [
`${index + 1}. ${item.title}`,
`- 原因:${item.why}`,
`- 练习:${item.howToPractice}`,
`- 达标:${item.successMetric}`,
].join("\n"))
.join("\n");
const drills = report.drills
.map((item) => [
`- ${item.name}${item.durationMinutes} 分钟)`,
` 目的:${item.purpose}`,
` 口令:${item.coachingCues.join(";")}`,
].join("\n"))
.join("\n");
return [
`## 动作概览`,
report.summary,
"",
`- 综合评分:${Math.round(report.overallScore)}/100`,
`- 置信度:${Math.round(report.confidence)}%`,
"",
"## 优先修正",
priorityFixes || "- 暂无",
"",
"## 推荐练习",
drills || "- 暂无",
"",
"## 风险提醒",
report.safetyRisks.length > 0 ? report.safetyRisks.map(item => `- ${item}`).join("\n") : "- 暂无明显风险",
"",
"## 下次训练重点",
report.nextSessionFocus.length > 0 ? report.nextSessionFocus.map(item => `- ${item}`).join("\n") : "- 保持当前节奏",
"",
"## 下次拍摄建议",
report.recommendedCaptureTips.length > 0 ? report.recommendedCaptureTips.map(item => `- ${item}`).join("\n") : "- 保持当前拍摄方式",
].join("\n");
}

22
server/publicUrl.ts 普通文件
查看文件

@@ -0,0 +1,22 @@
import { ENV } from "./_core/env";
function hasProtocol(value: string) {
return /^[a-z][a-z0-9+.-]*:\/\//i.test(value);
}
export function toPublicUrl(pathOrUrl: string) {
const value = pathOrUrl.trim();
if (!value) {
throw new Error("Public URL value is empty");
}
if (hasProtocol(value)) {
return value;
}
if (!ENV.appPublicBaseUrl) {
throw new Error("APP_PUBLIC_BASE_URL is required for externally accessible asset URLs");
}
return new URL(value.startsWith("/") ? value : `/${value}`, ENV.appPublicBaseUrl).toString();
}

查看文件

@@ -4,53 +4,34 @@ import { systemRouter } from "./_core/systemRouter";
import { publicProcedure, protectedProcedure, router } from "./_core/trpc";
import { z } from "zod";
import { sdk } from "./_core/sdk";
import { invokeLLM } from "./_core/llm";
import { storagePut } from "./storage";
import * as db from "./db";
import { nanoid } from "nanoid";
import {
normalizeAdjustedPlanResponse,
normalizeTrainingPlanResponse,
} from "./trainingPlan";
import { getRemoteMediaSession } from "./mediaService";
import { prepareCorrectionImageUrls } from "./taskWorker";
import { toPublicUrl } from "./publicUrl";
async function invokeStructuredPlan<T>(params: {
baseMessages: Array<{ role: "system" | "user"; content: string }>;
responseFormat: {
type: "json_schema";
json_schema: {
name: string;
strict: true;
schema: Record<string, unknown>;
};
};
parse: (content: unknown) => T;
async function enqueueTask(params: {
userId: number;
type: "media_finalize" | "training_plan_generate" | "training_plan_adjust" | "analysis_corrections" | "pose_correction_multimodal";
title: string;
payload: Record<string, unknown>;
message: string;
}) {
let lastError: unknown;
const taskId = nanoid();
await db.createBackgroundTask({
id: taskId,
userId: params.userId,
type: params.type,
title: params.title,
message: params.message,
payload: params.payload,
progress: 0,
maxAttempts: params.type === "media_finalize" ? 90 : 3,
});
for (let attempt = 0; attempt < 3; attempt++) {
const retryHint =
attempt === 0 || !(lastError instanceof Error)
? []
: [{
role: "user" as const,
content:
`上一次输出无法被系统解析,错误是:${lastError.message}` +
"请只返回一个合法、完整、可解析的 JSON 对象,不要包含额外说明、注释或 Markdown 代码块。",
}];
const response = await invokeLLM({
messages: [...params.baseMessages, ...retryHint],
response_format: params.responseFormat,
});
try {
return params.parse(response.choices[0]?.message?.content);
} catch (error) {
lastError = error;
}
}
throw lastError instanceof Error ? lastError : new Error("Failed to parse structured LLM response");
const task = await db.getBackgroundTaskById(taskId);
return { taskId, task };
}
export const appRouter = router({
@@ -104,86 +85,13 @@ export const appRouter = router({
focusAreas: z.array(z.string()).optional(),
}))
.mutation(async ({ ctx, input }) => {
const user = ctx.user;
// Get user's recent analyses for personalization
const analyses = await db.getUserAnalyses(user.id);
const recentScores = analyses.slice(0, 5).map(a => ({
score: a.overallScore,
issues: a.detectedIssues,
exerciseType: a.exerciseType,
shotCount: a.shotCount,
strokeConsistency: a.strokeConsistency,
footworkScore: a.footworkScore,
}));
const prompt = `你是一位网球教练。请为一位${
input.skillLevel === "beginner" ? "初级" : input.skillLevel === "intermediate" ? "中级" : "高级"
}水平的网球学员生成一个${input.durationDays}天的训练计划。
要求:
- 只需要球拍,不需要球场和球网
- 包含影子挥拍、墙壁练习、脚步移动、体能训练等
- 每天训练30-60分钟
${input.focusAreas?.length ? `- 重点关注: ${input.focusAreas.join(", ")}` : ""}
${recentScores.length > 0 ? `- 用户最近的分析数据: ${JSON.stringify(recentScores)}` : ""}
请返回JSON格式,包含每天的训练内容。`;
const parsed = await invokeStructuredPlan({
baseMessages: [
{ role: "system", content: "你是网球训练计划生成器。返回严格的JSON格式。" },
{ role: "user", content: prompt },
],
responseFormat: {
type: "json_schema",
json_schema: {
name: "training_plan",
strict: true,
schema: {
type: "object",
properties: {
title: { type: "string", description: "训练计划标题" },
exercises: {
type: "array",
items: {
type: "object",
properties: {
day: { type: "number" },
name: { type: "string" },
category: { type: "string" },
duration: { type: "number", description: "分钟" },
description: { type: "string" },
tips: { type: "string" },
sets: { type: "number" },
reps: { type: "number" },
},
required: ["day", "name", "category", "duration", "description", "tips", "sets", "reps"],
additionalProperties: false,
},
},
},
required: ["title", "exercises"],
additionalProperties: false,
},
},
},
parse: (content) => normalizeTrainingPlanResponse({
content,
fallbackTitle: `${input.durationDays}天训练计划`,
}),
return enqueueTask({
userId: ctx.user.id,
type: "training_plan_generate",
title: `${input.durationDays}天训练计划生成`,
message: "训练计划已加入后台队列",
payload: input,
});
const planId = await db.createTrainingPlan({
userId: user.id,
title: parsed.title,
skillLevel: input.skillLevel,
durationDays: input.durationDays,
exercises: parsed.exercises,
isActive: 1,
version: 1,
});
return { planId, plan: parsed };
}),
list: protectedProcedure.query(async ({ ctx }) => {
@@ -197,78 +105,15 @@ ${recentScores.length > 0 ? `- 用户最近的分析数据: ${JSON.stringify(rec
adjust: protectedProcedure
.input(z.object({ planId: z.number() }))
.mutation(async ({ ctx, input }) => {
const analyses = await db.getUserAnalyses(ctx.user.id);
const recentAnalyses = analyses.slice(0, 5);
const currentPlan = (await db.getUserTrainingPlans(ctx.user.id)).find(p => p.id === input.planId);
if (!currentPlan) throw new Error("Plan not found");
const prompt = `基于以下用户的姿势分析结果,调整训练计划:
当前计划: ${JSON.stringify(currentPlan.exercises)}
最近分析结果: ${JSON.stringify(recentAnalyses.map(a => ({
score: a.overallScore,
issues: a.detectedIssues,
corrections: a.corrections,
shotCount: a.shotCount,
strokeConsistency: a.strokeConsistency,
footworkScore: a.footworkScore,
fluidityScore: a.fluidityScore,
})))}
请根据分析结果调整训练计划,增加针对薄弱环节的训练,返回与原计划相同格式的JSON。`;
const parsed = await invokeStructuredPlan({
baseMessages: [
{ role: "system", content: "你是网球训练计划调整器。返回严格的JSON格式。" },
{ role: "user", content: prompt },
],
responseFormat: {
type: "json_schema",
json_schema: {
name: "adjusted_plan",
strict: true,
schema: {
type: "object",
properties: {
title: { type: "string" },
adjustmentNotes: { type: "string", description: "调整说明" },
exercises: {
type: "array",
items: {
type: "object",
properties: {
day: { type: "number" },
name: { type: "string" },
category: { type: "string" },
duration: { type: "number" },
description: { type: "string" },
tips: { type: "string" },
sets: { type: "number" },
reps: { type: "number" },
},
required: ["day", "name", "category", "duration", "description", "tips", "sets", "reps"],
additionalProperties: false,
},
},
},
required: ["title", "adjustmentNotes", "exercises"],
additionalProperties: false,
},
},
},
parse: (content) => normalizeAdjustedPlanResponse({
content,
fallbackTitle: currentPlan.title,
}),
return enqueueTask({
userId: ctx.user.id,
type: "training_plan_adjust",
title: `${currentPlan.title} 调整`,
message: "训练计划调整任务已加入后台队列",
payload: input,
});
await db.updateTrainingPlan(input.planId, {
exercises: parsed.exercises,
adjustmentNotes: parsed.adjustmentNotes,
version: (currentPlan.version || 1) + 1,
});
return { success: true, adjustmentNotes: parsed.adjustmentNotes };
}),
}),
@@ -287,19 +132,20 @@ ${recentScores.length > 0 ? `- 用户最近的分析数据: ${JSON.stringify(rec
const fileKey = `videos/${ctx.user.id}/${nanoid()}.${input.format}`;
const contentType = input.format === "webm" ? "video/webm" : "video/mp4";
const { url } = await storagePut(fileKey, fileBuffer, contentType);
const publicUrl = toPublicUrl(url);
const videoId = await db.createVideo({
userId: ctx.user.id,
title: input.title,
fileKey,
url,
url: publicUrl,
format: input.format,
fileSize: input.fileSize,
exerciseType: input.exerciseType || null,
analysisStatus: "pending",
});
return { videoId, url };
return { videoId, url: publicUrl };
}),
registerExternal: protectedProcedure
@@ -313,11 +159,12 @@ ${recentScores.length > 0 ? `- 用户最近的分析数据: ${JSON.stringify(rec
exerciseType: z.string().optional(),
}))
.mutation(async ({ ctx, input }) => {
const publicUrl = toPublicUrl(input.url);
const videoId = await db.createVideo({
userId: ctx.user.id,
title: input.title,
fileKey: input.fileKey,
url: input.url,
url: publicUrl,
format: input.format,
fileSize: input.fileSize ?? null,
duration: input.duration ?? null,
@@ -325,7 +172,7 @@ ${recentScores.length > 0 ? `- 用户最近的分析数据: ${JSON.stringify(rec
analysisStatus: "completed",
});
return { videoId, url: input.url };
return { videoId, url: publicUrl };
}),
list: protectedProcedure.query(async ({ ctx }) => {
@@ -399,32 +246,70 @@ ${recentScores.length > 0 ? `- 用户最近的分析数据: ${JSON.stringify(rec
poseMetrics: z.any(),
exerciseType: z.string(),
detectedIssues: z.any(),
imageUrls: z.array(z.string()).optional(),
imageDataUrls: z.array(z.string()).max(4).optional(),
}))
.mutation(async ({ input }) => {
const response = await invokeLLM({
messages: [
{
role: "system",
content: "你是一位网球动作分析员。根据MediaPipe姿势分析数据,给出具体的姿势矫正建议。用中文回答。",
},
{
role: "user",
content: `分析以下网球动作数据并给出矫正建议:
动作类型: ${input.exerciseType}
姿势指标: ${JSON.stringify(input.poseMetrics)}
检测到的问题: ${JSON.stringify(input.detectedIssues)}
请给出:
1. 每个问题的具体矫正方法
2. 推荐的练习动作
3. 需要注意的关键点`,
},
],
.mutation(async ({ ctx, input }) => {
const imageUrls = await prepareCorrectionImageUrls({
userId: ctx.user.id,
imageUrls: input.imageUrls,
imageDataUrls: input.imageDataUrls,
});
return {
corrections: response.choices[0]?.message?.content || "暂无建议",
};
return enqueueTask({
userId: ctx.user.id,
type: imageUrls.length > 0 ? "pose_correction_multimodal" : "analysis_corrections",
title: `${input.exerciseType} 动作纠正`,
message: imageUrls.length > 0 ? "多模态动作纠正任务已加入后台队列" : "动作纠正任务已加入后台队列",
payload: {
poseMetrics: input.poseMetrics,
exerciseType: input.exerciseType,
detectedIssues: input.detectedIssues,
imageUrls,
},
});
}),
}),
task: router({
list: protectedProcedure
.input(z.object({ limit: z.number().min(1).max(50).default(20) }).optional())
.query(async ({ ctx, input }) => {
return db.listUserBackgroundTasks(ctx.user.id, input?.limit ?? 20);
}),
get: protectedProcedure
.input(z.object({ taskId: z.string().min(1) }))
.query(async ({ ctx, input }) => {
return db.getUserBackgroundTaskById(ctx.user.id, input.taskId);
}),
retry: protectedProcedure
.input(z.object({ taskId: z.string().min(1) }))
.mutation(async ({ ctx, input }) => {
const task = await db.retryBackgroundTask(ctx.user.id, input.taskId);
return { task };
}),
createMediaFinalize: protectedProcedure
.input(z.object({
sessionId: z.string().min(1),
title: z.string().min(1).max(256),
exerciseType: z.string().optional(),
}))
.mutation(async ({ ctx, input }) => {
const session = await getRemoteMediaSession(input.sessionId);
if (session.userId !== String(ctx.user.id)) {
throw new Error("Media session not found");
}
return enqueueTask({
userId: ctx.user.id,
type: "media_finalize",
title: `${input.title} 归档`,
message: "录制文件归档任务已加入后台队列",
payload: input,
});
}),
}),

查看文件

@@ -37,4 +37,16 @@ describe("storage fallback", () => {
url: "/uploads/videos/test/sample.webm",
});
});
it("builds externally accessible URLs for local assets", async () => {
process.env.APP_PUBLIC_BASE_URL = "https://te.hao.work/";
const { toExternalAssetUrl } = await import("./storage");
expect(toExternalAssetUrl("/uploads/videos/test/sample.webm")).toBe(
"https://te.hao.work/uploads/videos/test/sample.webm"
);
expect(toExternalAssetUrl("https://cdn.example.com/demo.jpg")).toBe(
"https://cdn.example.com/demo.jpg"
);
});
});

查看文件

@@ -4,6 +4,7 @@
import { mkdir, readFile, writeFile } from "node:fs/promises";
import path from "node:path";
import { ENV } from './_core/env';
import { toPublicUrl } from "./publicUrl";
type StorageConfig = { baseUrl: string; apiKey: string };
@@ -141,3 +142,7 @@ export async function storageGet(relKey: string): Promise<{ key: string; url: st
url: await buildDownloadUrl(baseUrl, key, apiKey),
};
}
export function toExternalAssetUrl(pathOrUrl: string) {
return toPublicUrl(pathOrUrl);
}

470
server/taskWorker.ts 普通文件
查看文件

@@ -0,0 +1,470 @@
import { nanoid } from "nanoid";
import { ENV } from "./_core/env";
import { invokeLLM, type Message } from "./_core/llm";
import * as db from "./db";
import { getRemoteMediaSession } from "./mediaService";
import {
buildAdjustedTrainingPlanPrompt,
buildMultimodalCorrectionPrompt,
buildTextCorrectionPrompt,
buildTrainingPlanPrompt,
multimodalCorrectionSchema,
renderMultimodalCorrectionMarkdown,
} from "./prompts";
import { toPublicUrl } from "./publicUrl";
import { storagePut } from "./storage";
import {
normalizeAdjustedPlanResponse,
normalizeTrainingPlanResponse,
} from "./trainingPlan";
type TaskRow = Awaited<ReturnType<typeof db.getBackgroundTaskById>>;
type StructuredParams<T> = {
model?: string;
baseMessages: Array<{ role: "system" | "user"; content: string | Message["content"] }>;
responseFormat: {
type: "json_schema";
json_schema: {
name: string;
strict: true;
schema: Record<string, unknown>;
};
};
parse: (content: unknown) => T;
};
async function invokeStructured<T>(params: StructuredParams<T>) {
let lastError: unknown;
for (let attempt = 0; attempt < 3; attempt++) {
const retryHint =
attempt === 0 || !(lastError instanceof Error)
? []
: [{
role: "user" as const,
content:
`上一次输出无法被系统解析,错误是:${lastError.message}` +
"请只返回合法完整的 JSON 对象,不要附加 Markdown 或说明。",
}];
const response = await invokeLLM({
apiUrl: params.model === ENV.llmVisionModel ? ENV.llmVisionApiUrl : undefined,
apiKey: params.model === ENV.llmVisionModel ? ENV.llmVisionApiKey : undefined,
model: params.model,
messages: [...params.baseMessages, ...retryHint],
response_format: params.responseFormat,
});
try {
return params.parse(response.choices[0]?.message?.content);
} catch (error) {
lastError = error;
}
}
throw lastError instanceof Error ? lastError : new Error("Failed to parse structured LLM response");
}
function parseDataUrl(input: string) {
const match = input.match(/^data:(.+?);base64,(.+)$/);
if (!match) {
throw new Error("Invalid image data URL");
}
return {
contentType: match[1],
buffer: Buffer.from(match[2], "base64"),
};
}
async function persistInlineImages(userId: number, imageDataUrls: string[]) {
const persistedUrls: string[] = [];
for (let index = 0; index < imageDataUrls.length; index++) {
const { contentType, buffer } = parseDataUrl(imageDataUrls[index]);
const extension = contentType.includes("png") ? "png" : "jpg";
const key = `analysis-images/${userId}/${nanoid()}.${extension}`;
const uploaded = await storagePut(key, buffer, contentType);
persistedUrls.push(toPublicUrl(uploaded.url));
}
return persistedUrls;
}
export async function prepareCorrectionImageUrls(input: {
userId: number;
imageUrls?: string[];
imageDataUrls?: string[];
}) {
const directUrls = (input.imageUrls ?? []).map((item) => toPublicUrl(item));
const uploadedUrls = input.imageDataUrls?.length
? await persistInlineImages(input.userId, input.imageDataUrls)
: [];
return [...directUrls, ...uploadedUrls];
}
async function runTrainingPlanGenerateTask(task: NonNullable<TaskRow>) {
const payload = task.payload as {
skillLevel: "beginner" | "intermediate" | "advanced";
durationDays: number;
focusAreas?: string[];
};
const analyses = await db.getUserAnalyses(task.userId);
const recentScores = analyses.slice(0, 5).map((analysis) => ({
score: analysis.overallScore ?? null,
issues: analysis.detectedIssues,
exerciseType: analysis.exerciseType ?? null,
shotCount: analysis.shotCount ?? null,
strokeConsistency: analysis.strokeConsistency ?? null,
footworkScore: analysis.footworkScore ?? null,
}));
const parsed = await invokeStructured({
baseMessages: [
{ role: "system", content: "你是网球训练计划生成器。返回严格的 JSON 格式。" },
{
role: "user",
content: buildTrainingPlanPrompt({
...payload,
recentScores,
}),
},
],
responseFormat: {
type: "json_schema",
json_schema: {
name: "training_plan",
strict: true,
schema: {
type: "object",
properties: {
title: { type: "string" },
exercises: {
type: "array",
items: {
type: "object",
properties: {
day: { type: "number" },
name: { type: "string" },
category: { type: "string" },
duration: { type: "number" },
description: { type: "string" },
tips: { type: "string" },
sets: { type: "number" },
reps: { type: "number" },
},
required: ["day", "name", "category", "duration", "description", "tips", "sets", "reps"],
additionalProperties: false,
},
},
},
required: ["title", "exercises"],
additionalProperties: false,
},
},
},
parse: (content) => normalizeTrainingPlanResponse({
content,
fallbackTitle: `${payload.durationDays}天训练计划`,
}),
});
const planId = await db.createTrainingPlan({
userId: task.userId,
title: parsed.title,
skillLevel: payload.skillLevel,
durationDays: payload.durationDays,
exercises: parsed.exercises,
isActive: 1,
version: 1,
});
return {
kind: "training_plan_generate" as const,
planId,
plan: parsed,
};
}
async function runTrainingPlanAdjustTask(task: NonNullable<TaskRow>) {
const payload = task.payload as { planId: number };
const analyses = await db.getUserAnalyses(task.userId);
const recentAnalyses = analyses.slice(0, 5);
const currentPlan = (await db.getUserTrainingPlans(task.userId)).find((plan) => plan.id === payload.planId);
if (!currentPlan) {
throw new Error("Plan not found");
}
const parsed = await invokeStructured({
baseMessages: [
{ role: "system", content: "你是网球训练计划调整器。返回严格的 JSON 格式。" },
{
role: "user",
content: buildAdjustedTrainingPlanPrompt({
currentExercises: currentPlan.exercises,
recentAnalyses: recentAnalyses.map((analysis) => ({
score: analysis.overallScore ?? null,
issues: analysis.detectedIssues,
corrections: analysis.corrections,
shotCount: analysis.shotCount ?? null,
strokeConsistency: analysis.strokeConsistency ?? null,
footworkScore: analysis.footworkScore ?? null,
fluidityScore: analysis.fluidityScore ?? null,
})),
}),
},
],
responseFormat: {
type: "json_schema",
json_schema: {
name: "adjusted_plan",
strict: true,
schema: {
type: "object",
properties: {
title: { type: "string" },
adjustmentNotes: { type: "string" },
exercises: {
type: "array",
items: {
type: "object",
properties: {
day: { type: "number" },
name: { type: "string" },
category: { type: "string" },
duration: { type: "number" },
description: { type: "string" },
tips: { type: "string" },
sets: { type: "number" },
reps: { type: "number" },
},
required: ["day", "name", "category", "duration", "description", "tips", "sets", "reps"],
additionalProperties: false,
},
},
},
required: ["title", "adjustmentNotes", "exercises"],
additionalProperties: false,
},
},
},
parse: (content) => normalizeAdjustedPlanResponse({
content,
fallbackTitle: currentPlan.title,
}),
});
await db.updateTrainingPlan(payload.planId, {
exercises: parsed.exercises,
adjustmentNotes: parsed.adjustmentNotes,
version: (currentPlan.version || 1) + 1,
});
return {
kind: "training_plan_adjust" as const,
planId: payload.planId,
plan: parsed,
adjustmentNotes: parsed.adjustmentNotes,
};
}
async function runTextCorrectionTask(task: NonNullable<TaskRow>) {
const payload = task.payload as {
exerciseType: string;
poseMetrics: unknown;
detectedIssues: unknown;
};
return createTextCorrectionResult(payload);
}
async function createTextCorrectionResult(payload: {
exerciseType: string;
poseMetrics: unknown;
detectedIssues: unknown;
}) {
const response = await invokeLLM({
messages: [
{
role: "system",
content: "你是一位专业网球技术教练。输出中文 Markdown,内容具体、克制、可执行。",
},
{
role: "user",
content: buildTextCorrectionPrompt(payload),
},
],
});
return {
kind: "analysis_corrections" as const,
corrections: response.choices[0]?.message?.content || "暂无建议",
};
}
async function runMultimodalCorrectionTask(task: NonNullable<TaskRow>) {
const payload = task.payload as {
exerciseType: string;
poseMetrics: unknown;
detectedIssues: unknown;
imageUrls: string[];
};
try {
const report = await invokeStructured({
model: ENV.llmVisionModel,
baseMessages: [
{ role: "system", content: "你是专业网球教练。请基于图片和结构化姿态指标输出严格 JSON。" },
{
role: "user",
content: [
{ type: "text", text: buildMultimodalCorrectionPrompt({
exerciseType: payload.exerciseType,
poseMetrics: payload.poseMetrics,
detectedIssues: payload.detectedIssues,
imageCount: payload.imageUrls.length,
}) },
...payload.imageUrls.map((url) => ({
type: "image_url" as const,
image_url: {
url,
detail: "high" as const,
},
})),
],
},
],
responseFormat: {
type: "json_schema",
json_schema: {
name: "pose_correction_multimodal",
strict: true,
schema: multimodalCorrectionSchema,
},
},
parse: (content) => {
if (typeof content === "string") {
return JSON.parse(content);
}
return content as Record<string, unknown>;
},
});
return {
kind: "pose_correction_multimodal" as const,
imageUrls: payload.imageUrls,
report,
corrections: renderMultimodalCorrectionMarkdown(report as Parameters<typeof renderMultimodalCorrectionMarkdown>[0]),
visionStatus: "ok" as const,
};
} catch (error) {
const fallback = await createTextCorrectionResult(payload);
return {
kind: "pose_correction_multimodal" as const,
imageUrls: payload.imageUrls,
report: null,
corrections: fallback.corrections,
visionStatus: "fallback" as const,
warning: error instanceof Error ? error.message : "Vision model unavailable",
};
}
}
async function runMediaFinalizeTask(task: NonNullable<TaskRow>) {
const payload = task.payload as {
sessionId: string;
title: string;
exerciseType?: string;
};
const session = await getRemoteMediaSession(payload.sessionId);
if (session.userId !== String(task.userId)) {
throw new Error("Media session does not belong to the task user");
}
if (session.archiveStatus === "queued") {
await db.rescheduleBackgroundTask(task.id, {
progress: 45,
message: "录制文件已入队,等待归档",
delayMs: 4_000,
});
return null;
}
if (session.archiveStatus === "processing") {
await db.rescheduleBackgroundTask(task.id, {
progress: 78,
message: "录制文件正在整理与转码",
delayMs: 4_000,
});
return null;
}
if (session.archiveStatus === "failed") {
throw new Error(session.lastError || "Media archive failed");
}
if (!session.playback.ready) {
await db.rescheduleBackgroundTask(task.id, {
progress: 70,
message: "等待回放文件就绪",
delayMs: 4_000,
});
return null;
}
const preferredUrl = session.playback.mp4Url || session.playback.webmUrl;
const format = session.playback.mp4Url ? "mp4" : "webm";
if (!preferredUrl) {
throw new Error("Media session did not expose a playback URL");
}
const fileKey = `media/sessions/${session.id}/recording.${format}`;
const existing = await db.getVideoByFileKey(task.userId, fileKey);
if (existing) {
return {
kind: "media_finalize" as const,
sessionId: session.id,
videoId: existing.id,
url: existing.url,
fileKey,
format,
};
}
const publicUrl = toPublicUrl(preferredUrl);
const videoId = await db.createVideo({
userId: task.userId,
title: payload.title || session.title,
fileKey,
url: publicUrl,
format,
fileSize: format === "mp4" ? (session.playback.mp4Size ?? null) : (session.playback.webmSize ?? null),
duration: null,
exerciseType: payload.exerciseType || "recording",
analysisStatus: "completed",
});
return {
kind: "media_finalize" as const,
sessionId: session.id,
videoId,
url: publicUrl,
fileKey,
format,
};
}
export async function processBackgroundTask(task: NonNullable<TaskRow>) {
switch (task.type) {
case "training_plan_generate":
return runTrainingPlanGenerateTask(task);
case "training_plan_adjust":
return runTrainingPlanAdjustTask(task);
case "analysis_corrections":
return runTextCorrectionTask(task);
case "pose_correction_multimodal":
return runMultimodalCorrectionTask(task);
case "media_finalize":
return runMediaFinalizeTask(task);
default:
throw new Error(`Unsupported task type: ${String(task.type)}`);
}
}

47
server/worker.ts 普通文件
查看文件

@@ -0,0 +1,47 @@
import "dotenv/config";
import { ENV } from "./_core/env";
import * as db from "./db";
import { processBackgroundTask } from "./taskWorker";
const workerId = `app-worker-${process.pid}`;
function sleep(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function workOnce() {
await db.requeueStaleBackgroundTasks(new Date(Date.now() - ENV.backgroundTaskStaleMs));
const task = await db.claimNextBackgroundTask(workerId);
if (!task) {
return false;
}
try {
const result = await processBackgroundTask(task);
if (result !== null) {
await db.completeBackgroundTask(task.id, result, "任务执行完成");
}
} catch (error) {
const message = error instanceof Error ? error.message : "Unknown background task error";
await db.failBackgroundTask(task.id, message);
console.error(`[worker] task ${task.id} failed:`, error);
}
return true;
}
async function main() {
console.log(`[worker] ${workerId} started`);
for (;;) {
const hasWorked = await workOnce();
if (!hasWorked) {
await sleep(ENV.backgroundTaskPollMs);
}
}
}
main().catch((error) => {
console.error("[worker] fatal error", error);
process.exit(1);
});