Harden async task flows and enhance analysis tooling

server/_core/env.ts

@@ -51,7 +51,13 @@ export const ENV = {
   llmMaxTokens: parseInteger(process.env.LLM_MAX_TOKENS, 32768),
   llmEnableThinking: parseBoolean(process.env.LLM_ENABLE_THINKING, false),
   llmThinkingBudget: parseInteger(process.env.LLM_THINKING_BUDGET, 128),
+  llmTimeoutMs: parseInteger(process.env.LLM_TIMEOUT_MS, 45000),
+  llmRetryCount: parseInteger(process.env.LLM_RETRY_COUNT, 1),
   mediaServiceUrl: process.env.MEDIA_SERVICE_URL ?? "",
+  mediaFetchTimeoutMs: parseInteger(process.env.MEDIA_FETCH_TIMEOUT_MS, 12000),
+  mediaFetchRetryCount: parseInteger(process.env.MEDIA_FETCH_RETRY_COUNT, 2),
+  youtubeApiKey: process.env.YOUTUBE_API_KEY ?? "",
   backgroundTaskPollMs: parseInteger(process.env.BACKGROUND_TASK_POLL_MS, 3000),
   backgroundTaskStaleMs: parseInteger(process.env.BACKGROUND_TASK_STALE_MS, 300000),
+  backgroundTaskHeartbeatMs: parseInteger(process.env.BACKGROUND_TASK_HEARTBEAT_MS, 5000),
 };

server/_core/fetch.ts

@@ -0,0 +1,85 @@
+type FetchRetryOptions = {
+  timeoutMs: number;
+  retries?: number;
+  retryStatuses?: number[];
+  retryMethods?: string[];
+  baseDelayMs?: number;
+};
+
+const DEFAULT_RETRY_STATUSES = [408, 425, 429, 502, 503, 504];
+
+function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function shouldRetryResponse(method: string, response: Response, options: FetchRetryOptions) {
+  const allowedMethods = options.retryMethods ?? ["GET", "HEAD"];
+  const retryStatuses = options.retryStatuses ?? DEFAULT_RETRY_STATUSES;
+  return allowedMethods.includes(method) && retryStatuses.includes(response.status);
+}
+
+function shouldRetryError(method: string, error: unknown, options: FetchRetryOptions) {
+  const allowedMethods = options.retryMethods ?? ["GET", "HEAD"];
+  if (!allowedMethods.includes(method)) {
+    return false;
+  }
+
+  if (error instanceof Error) {
+    return error.name === "AbortError" || error.name === "TimeoutError" || error.message.includes("fetch");
+  }
+
+  return false;
+}
+
+export async function fetchWithTimeout(input: string | URL, init: RequestInit | undefined, options: FetchRetryOptions) {
+  const method = (init?.method ?? "GET").toUpperCase();
+  const retries = Math.max(0, options.retries ?? 0);
+  const baseDelayMs = Math.max(150, options.baseDelayMs ?? 350);
+  let lastError: unknown;
+
+  for (let attempt = 0; attempt <= retries; attempt += 1) {
+    const controller = new AbortController();
+    const upstreamSignal = init?.signal;
+    let didTimeout = false;
+
+    const timeout = setTimeout(() => {
+      didTimeout = true;
+      controller.abort();
+    }, options.timeoutMs);
+
+    const abortHandler = () => controller.abort();
+    upstreamSignal?.addEventListener("abort", abortHandler, { once: true });
+
+    try {
+      const response = await fetch(input, {
+        ...init,
+        signal: controller.signal,
+      });
+
+      if (attempt < retries && shouldRetryResponse(method, response, options)) {
+        await response.text().catch(() => undefined);
+        await sleep(baseDelayMs * (attempt + 1));
+        continue;
+      }
+
+      return response;
+    } catch (error) {
+      if (didTimeout) {
+        lastError = new Error(`Request timed out after ${options.timeoutMs}ms`);
+      } else {
+        lastError = error;
+      }
+
+      if (attempt >= retries || !shouldRetryError(method, lastError, options)) {
+        throw lastError instanceof Error ? lastError : new Error("Request failed");
+      }
+
+      await sleep(baseDelayMs * (attempt + 1));
+    } finally {
+      clearTimeout(timeout);
+      upstreamSignal?.removeEventListener("abort", abortHandler);
+    }
+  }
+
+  throw lastError instanceof Error ? lastError : new Error("Request failed");
+}

server/_core/llm.ts

@@ -1,4 +1,5 @@
 import { ENV } from "./env";
+import { fetchWithTimeout } from "./fetch";
 
 export type Role = "system" | "user" | "assistant" | "tool" | "function";
 
@@ -323,13 +324,17 @@ export async function invokeLLM(params: InvokeParams): Promise<InvokeResult> {
     payload.response_format = normalizedResponseFormat;
   }
 
-  const response = await fetch(resolveApiUrl(apiUrl), {
+  const response = await fetchWithTimeout(resolveApiUrl(apiUrl), {
     method: "POST",
     headers: {
       "content-type": "application/json",
       authorization: `Bearer ${apiKey || ENV.llmApiKey}`,
     },
     body: JSON.stringify(payload),
+  }, {
+    timeoutMs: ENV.llmTimeoutMs,
+    retries: ENV.llmRetryCount,
+    retryMethods: ["POST"],
   });
 
   if (!response.ok) {