import { nanoid } from "nanoid"; import { ENV } from "./_core/env"; import { invokeLLM, type Message } from "./_core/llm"; import * as db from "./db"; import { getRemoteMediaSession } from "./mediaService"; import { buildAdjustedTrainingPlanPrompt, buildMultimodalCorrectionPrompt, buildTextCorrectionPrompt, buildTrainingPlanPrompt, multimodalCorrectionSchema, renderMultimodalCorrectionMarkdown, } from "./prompts"; import { toPublicUrl } from "./publicUrl"; import { storagePut } from "./storage"; import { normalizeAdjustedPlanResponse, normalizeTrainingPlanResponse, } from "./trainingPlan"; type TaskRow = Awaited>; type StructuredParams = { model?: string; baseMessages: Array<{ role: "system" | "user"; content: string | Message["content"] }>; responseFormat: { type: "json_schema"; json_schema: { name: string; strict: true; schema: Record; }; }; parse: (content: unknown) => T; }; async function invokeStructured(params: StructuredParams) { let lastError: unknown; for (let attempt = 0; attempt < 3; attempt++) { const retryHint = attempt === 0 || !(lastError instanceof Error) ? [] : [{ role: "user" as const, content: `上一次输出无法被系统解析,错误是:${lastError.message}。` + "请只返回合法完整的 JSON 对象,不要附加 Markdown 或说明。", }]; const response = await invokeLLM({ apiUrl: params.model === ENV.llmVisionModel ? ENV.llmVisionApiUrl : undefined, apiKey: params.model === ENV.llmVisionModel ? ENV.llmVisionApiKey : undefined, model: params.model, messages: [...params.baseMessages, ...retryHint], response_format: params.responseFormat, }); try { return params.parse(response.choices[0]?.message?.content); } catch (error) { lastError = error; } } throw lastError instanceof Error ? lastError : new Error("Failed to parse structured LLM response"); } function contentToPlainText(content: Message["content"]) { if (typeof content === "string") { return content; } const parts = Array.isArray(content) ? content : [content]; return parts .map((part) => { if (typeof part === "string") { return part; } if (part.type === "text") { return part.text; } if (part.type === "image_url") { return `[image] ${part.image_url.url}`; } if (part.type === "file_url") { return `[file] ${part.file_url.url}`; } return ""; }) .filter(Boolean) .join("\n"); } function parseDataUrl(input: string) { const match = input.match(/^data:(.+?);base64,(.+)$/); if (!match) { throw new Error("Invalid image data URL"); } return { contentType: match[1], buffer: Buffer.from(match[2], "base64"), }; } async function persistInlineImages(userId: number, imageDataUrls: string[]) { const persistedUrls: string[] = []; for (let index = 0; index < imageDataUrls.length; index++) { const { contentType, buffer } = parseDataUrl(imageDataUrls[index]); const extension = contentType.includes("png") ? "png" : "jpg"; const key = `analysis-images/${userId}/${nanoid()}.${extension}`; const uploaded = await storagePut(key, buffer, contentType); persistedUrls.push(toPublicUrl(uploaded.url)); } return persistedUrls; } export async function prepareCorrectionImageUrls(input: { userId: number; imageUrls?: string[]; imageDataUrls?: string[]; }) { const directUrls = (input.imageUrls ?? []).map((item) => toPublicUrl(item)); const uploadedUrls = input.imageDataUrls?.length ? await persistInlineImages(input.userId, input.imageDataUrls) : []; return [...directUrls, ...uploadedUrls]; } async function runTrainingPlanGenerateTask(task: NonNullable) { const payload = task.payload as { skillLevel: "beginner" | "intermediate" | "advanced"; durationDays: number; focusAreas?: string[]; }; const analyses = await db.getUserAnalyses(task.userId); const recentScores = analyses.slice(0, 5).map((analysis) => ({ score: analysis.overallScore ?? null, issues: analysis.detectedIssues, exerciseType: analysis.exerciseType ?? null, shotCount: analysis.shotCount ?? null, strokeConsistency: analysis.strokeConsistency ?? null, footworkScore: analysis.footworkScore ?? null, })); const parsed = await invokeStructured({ baseMessages: [ { role: "system", content: "你是网球训练计划生成器。返回严格的 JSON 格式。" }, { role: "user", content: buildTrainingPlanPrompt({ ...payload, recentScores, }), }, ], responseFormat: { type: "json_schema", json_schema: { name: "training_plan", strict: true, schema: { type: "object", properties: { title: { type: "string" }, exercises: { type: "array", items: { type: "object", properties: { day: { type: "number" }, name: { type: "string" }, category: { type: "string" }, duration: { type: "number" }, description: { type: "string" }, tips: { type: "string" }, sets: { type: "number" }, reps: { type: "number" }, }, required: ["day", "name", "category", "duration", "description", "tips", "sets", "reps"], additionalProperties: false, }, }, }, required: ["title", "exercises"], additionalProperties: false, }, }, }, parse: (content) => normalizeTrainingPlanResponse({ content, fallbackTitle: `${payload.durationDays}天训练计划`, }), }); const planId = await db.createTrainingPlan({ userId: task.userId, title: parsed.title, skillLevel: payload.skillLevel, durationDays: payload.durationDays, exercises: parsed.exercises, isActive: 1, version: 1, }); return { kind: "training_plan_generate" as const, planId, plan: parsed, }; } async function runTrainingPlanAdjustTask(task: NonNullable) { const payload = task.payload as { planId: number }; const analyses = await db.getUserAnalyses(task.userId); const recentAnalyses = analyses.slice(0, 5); const currentPlan = (await db.getUserTrainingPlans(task.userId)).find((plan) => plan.id === payload.planId); if (!currentPlan) { throw new Error("Plan not found"); } const parsed = await invokeStructured({ baseMessages: [ { role: "system", content: "你是网球训练计划调整器。返回严格的 JSON 格式。" }, { role: "user", content: buildAdjustedTrainingPlanPrompt({ currentExercises: currentPlan.exercises, recentAnalyses: recentAnalyses.map((analysis) => ({ score: analysis.overallScore ?? null, issues: analysis.detectedIssues, corrections: analysis.corrections, shotCount: analysis.shotCount ?? null, strokeConsistency: analysis.strokeConsistency ?? null, footworkScore: analysis.footworkScore ?? null, fluidityScore: analysis.fluidityScore ?? null, })), }), }, ], responseFormat: { type: "json_schema", json_schema: { name: "adjusted_plan", strict: true, schema: { type: "object", properties: { title: { type: "string" }, adjustmentNotes: { type: "string" }, exercises: { type: "array", items: { type: "object", properties: { day: { type: "number" }, name: { type: "string" }, category: { type: "string" }, duration: { type: "number" }, description: { type: "string" }, tips: { type: "string" }, sets: { type: "number" }, reps: { type: "number" }, }, required: ["day", "name", "category", "duration", "description", "tips", "sets", "reps"], additionalProperties: false, }, }, }, required: ["title", "adjustmentNotes", "exercises"], additionalProperties: false, }, }, }, parse: (content) => normalizeAdjustedPlanResponse({ content, fallbackTitle: currentPlan.title, }), }); await db.updateTrainingPlan(payload.planId, { exercises: parsed.exercises, adjustmentNotes: parsed.adjustmentNotes, version: (currentPlan.version || 1) + 1, }); return { kind: "training_plan_adjust" as const, planId: payload.planId, plan: parsed, adjustmentNotes: parsed.adjustmentNotes, }; } async function runTextCorrectionTask(task: NonNullable) { const payload = task.payload as { exerciseType: string; poseMetrics: unknown; detectedIssues: unknown; }; return createTextCorrectionResult(payload); } async function createTextCorrectionResult(payload: { exerciseType: string; poseMetrics: unknown; detectedIssues: unknown; }) { const response = await invokeLLM({ messages: [ { role: "system", content: "你是一位专业网球技术教练。输出中文 Markdown,内容具体、克制、可执行。", }, { role: "user", content: buildTextCorrectionPrompt(payload), }, ], }); return { kind: "analysis_corrections" as const, corrections: contentToPlainText(response.choices[0]?.message?.content || "暂无建议"), }; } async function runMultimodalCorrectionTask(task: NonNullable) { const payload = task.payload as { exerciseType: string; poseMetrics: unknown; detectedIssues: unknown; imageUrls: string[]; }; try { const report = await invokeStructured({ model: ENV.llmVisionModel, baseMessages: [ { role: "system", content: "你是专业网球教练。请基于图片和结构化姿态指标输出严格 JSON。" }, { role: "user", content: [ { type: "text", text: buildMultimodalCorrectionPrompt({ exerciseType: payload.exerciseType, poseMetrics: payload.poseMetrics, detectedIssues: payload.detectedIssues, imageCount: payload.imageUrls.length, }) }, ...payload.imageUrls.map((url) => ({ type: "image_url" as const, image_url: { url, detail: "high" as const, }, })), ], }, ], responseFormat: { type: "json_schema", json_schema: { name: "pose_correction_multimodal", strict: true, schema: multimodalCorrectionSchema, }, }, parse: (content) => { if (typeof content === "string") { return JSON.parse(content); } return content as Record; }, }); const result = { kind: "pose_correction_multimodal" as const, imageUrls: payload.imageUrls, report, corrections: renderMultimodalCorrectionMarkdown(report as Parameters[0]), visionStatus: "ok" as const, }; await db.completeVisionTestRun(task.id, { visionStatus: "ok", summary: (report as { summary?: string }).summary ?? null, corrections: result.corrections, report, warning: null, }); return result; } catch (error) { const fallback = await createTextCorrectionResult(payload); const result = { kind: "pose_correction_multimodal" as const, imageUrls: payload.imageUrls, report: null, corrections: fallback.corrections, visionStatus: "fallback" as const, warning: error instanceof Error ? error.message : "Vision model unavailable", }; await db.completeVisionTestRun(task.id, { visionStatus: "fallback", summary: null, corrections: result.corrections, report: null, warning: result.warning, }); return result; } } async function runMediaFinalizeTask(task: NonNullable) { const payload = task.payload as { sessionId: string; title: string; exerciseType?: string; }; const session = await getRemoteMediaSession(payload.sessionId); if (session.userId !== String(task.userId)) { throw new Error("Media session does not belong to the task user"); } if (session.archiveStatus === "queued") { await db.rescheduleBackgroundTask(task.id, { progress: 45, message: "录制文件已入队,等待归档", delayMs: 4_000, }); return null; } if (session.archiveStatus === "processing") { await db.rescheduleBackgroundTask(task.id, { progress: 78, message: "录制文件正在整理与转码", delayMs: 4_000, }); return null; } if (session.archiveStatus === "failed") { throw new Error(session.lastError || "Media archive failed"); } if (!session.playback.ready) { await db.rescheduleBackgroundTask(task.id, { progress: 70, message: "等待回放文件就绪", delayMs: 4_000, }); return null; } const preferredUrl = session.playback.mp4Url || session.playback.webmUrl; const format = session.playback.mp4Url ? "mp4" : "webm"; if (!preferredUrl) { throw new Error("Media session did not expose a playback URL"); } const fileKey = `media/sessions/${session.id}/recording.${format}`; const existing = await db.getVideoByFileKey(task.userId, fileKey); if (existing) { return { kind: "media_finalize" as const, sessionId: session.id, videoId: existing.id, url: existing.url, fileKey, format, }; } const publicUrl = toPublicUrl(preferredUrl); const videoId = await db.createVideo({ userId: task.userId, title: payload.title || session.title, fileKey, url: publicUrl, format, fileSize: format === "mp4" ? (session.playback.mp4Size ?? null) : (session.playback.webmSize ?? null), duration: null, exerciseType: payload.exerciseType || "recording", analysisStatus: "completed", }); return { kind: "media_finalize" as const, sessionId: session.id, videoId, url: publicUrl, fileKey, format, }; } export async function processBackgroundTask(task: NonNullable) { switch (task.type) { case "training_plan_generate": return runTrainingPlanGenerateTask(task); case "training_plan_adjust": return runTrainingPlanAdjustTask(task); case "analysis_corrections": return runTextCorrectionTask(task); case "pose_correction_multimodal": return runMultimodalCorrectionTask(task); case "media_finalize": return runMediaFinalizeTask(task); default: throw new Error(`Unsupported task type: ${String(task.type)}`); } }