89 行
2.7 KiB
TypeScript
89 行
2.7 KiB
TypeScript
import "dotenv/config";
|
|
import { ENV } from "./_core/env";
|
|
import * as db from "./db";
|
|
import { processBackgroundTask } from "./taskWorker";
|
|
|
|
const workerId = `app-worker-${process.pid}`;
|
|
|
|
function sleep(ms: number) {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
function isRetriableBackgroundError(error: unknown) {
|
|
if (!(error instanceof Error)) return false;
|
|
return (
|
|
error.message.startsWith("Request timed out after ") ||
|
|
error.message.includes("fetch failed") ||
|
|
error.message.includes("ECONNRESET") ||
|
|
error.message.includes("ETIMEDOUT") ||
|
|
error.message.includes("429") ||
|
|
error.message.includes("502") ||
|
|
error.message.includes("503") ||
|
|
error.message.includes("504")
|
|
);
|
|
}
|
|
|
|
async function workOnce() {
|
|
await db.failExhaustedBackgroundTasks();
|
|
await db.requeueStaleBackgroundTasks(new Date(Date.now() - ENV.backgroundTaskStaleMs));
|
|
|
|
const task = await db.claimNextBackgroundTask(workerId);
|
|
if (!task) {
|
|
return false;
|
|
}
|
|
|
|
const heartbeatTimer = setInterval(() => {
|
|
void db.heartbeatBackgroundTask(task.id, workerId).catch((error) => {
|
|
console.error(`[worker] heartbeat failed for ${task.id}:`, error);
|
|
});
|
|
}, ENV.backgroundTaskHeartbeatMs);
|
|
|
|
try {
|
|
const result = await processBackgroundTask(task);
|
|
if (result !== null) {
|
|
await db.completeBackgroundTask(task.id, result, "任务执行完成");
|
|
}
|
|
} catch (error) {
|
|
const message = error instanceof Error ? error.message : "Unknown background task error";
|
|
if (isRetriableBackgroundError(error) && task.attempts < task.maxAttempts) {
|
|
const nextAttempt = task.attempts + 1;
|
|
const delayMs = Math.min(30_000, 5_000 * task.attempts);
|
|
await db.rescheduleBackgroundTask(task.id, {
|
|
progress: 15,
|
|
message: `请求超时，已自动重试（第 ${nextAttempt}/${task.maxAttempts} 次尝试）`,
|
|
error: message,
|
|
delayMs,
|
|
});
|
|
console.warn(`[worker] task ${task.id} rescheduled after retriable error:`, error);
|
|
} else {
|
|
await db.failBackgroundTask(task.id, message);
|
|
await db.failVisionTestRun(task.id, message);
|
|
console.error(`[worker] task ${task.id} failed:`, error);
|
|
}
|
|
} finally {
|
|
clearInterval(heartbeatTimer);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
async function main() {
|
|
console.log(`[worker] ${workerId} started`);
|
|
for (;;) {
|
|
try {
|
|
const hasWorked = await workOnce();
|
|
if (!hasWorked) {
|
|
await sleep(ENV.backgroundTaskPollMs);
|
|
}
|
|
} catch (error) {
|
|
console.error("[worker] loop error", error);
|
|
await sleep(Math.max(ENV.backgroundTaskPollMs, 3_000));
|
|
}
|
|
}
|
|
}
|
|
|
|
main().catch((error) => {
|
|
console.error("[worker] fatal error", error);
|
|
process.exit(1);
|
|
});
|