import { appendEvent, expireStaleLeases, getEnv, markOfflineWorkers, requeueRetrying, transition, } from "@autoscholar/core"; import { type Db, researchRuns } from "@autoscholar/db"; import type { Logger } from "@autoscholar/observability"; import { publishRun } from "@autoscholar/publisher"; import { evaluateRun } from "@autoscholar/qa"; import type { FilesystemStorage } from "@autoscholar/storage"; import { and, eq, inArray, lt, sql } from "drizzle-orm"; export interface TickReport { expiredLeases: string[]; requeued: number; offlineWorkers: number; evaluated: string[]; published: string[]; timedOut: string[]; errors: Array<{ runId?: string; step: string; error: string }>; } /** One scheduler pass. Every step is idempotent, so overlapping or restarted schedulers are safe. */ export async function schedulerTick(db: Db, storage: FilesystemStorage, log: Logger): Promise { const env = getEnv(); const report: TickReport = { expiredLeases: [], requeued: 0, offlineWorkers: 0, evaluated: [], published: [], timedOut: [], errors: [], }; try { report.expiredLeases = await expireStaleLeases(db); } catch (e) { report.errors.push({ step: "expireStaleLeases", error: (e as Error).message }); } try { report.requeued = await requeueRetrying(db); } catch (e) { report.errors.push({ step: "requeueRetrying", error: (e as Error).message }); } try { report.offlineWorkers = await markOfflineWorkers(db); } catch (e) { report.errors.push({ step: "markOfflineWorkers", error: (e as Error).message }); } // runtime budget enforced outside the LLM: request cancellation when wall clock exceeds the cap (+10% grace) try { const running = await db .select() .from(researchRuns) .where( and( inArray(researchRuns.status, ["running", "provisioning"]), eq(researchRuns.cancelRequested, false), ), ); for (const r of running) { if (!r.startedAt) continue; const limitMs = r.maxRuntimeMinutes * 60_000 * 1.1; if (Date.now() - r.startedAt.getTime() > limitMs) { await db.transaction(async (tx) => { await tx .update(researchRuns) .set({ cancelRequested: true, updatedAt: new Date() }) .where(eq(researchRuns.id, r.id)); await transition(tx, r.id, "cancelling", { actor: "scheduler", reason: `runtime cap ${r.maxRuntimeMinutes} min exceeded`, }); }); report.timedOut.push(r.id); } } } catch (e) { report.errors.push({ step: "runtimeCap", error: (e as Error).message }); } // Stage 9: quality decisions const toEvaluate = await db .select({ id: researchRuns.id }) .from(researchRuns) .where(eq(researchRuns.status, "quality_check")); for (const r of toEvaluate) { try { const { nextStatus } = await evaluateRun(db, storage, r.id); report.evaluated.push(`${r.id}:${nextStatus}`); } catch (e) { report.errors.push({ runId: r.id, step: "evaluate", error: (e as Error).message }); log.error({ runId: r.id, err: (e as Error).message }, "quality evaluation crashed"); try { await db.transaction((tx) => appendEvent(tx, r.id, { kind: "qa.error", level: "error", message: `quality evaluation error: ${(e as Error).message}`.slice(0, 2000), at: new Date().toISOString(), }), ); } catch { /* ignore */ } } } // Stage 10: publication for runs in `publishing` (auto-publish or admin-approved); retried every tick until done if (env.PUBLISHING_ENABLED === 1) { const toPublish = await db .select({ id: researchRuns.id }) .from(researchRuns) .where(eq(researchRuns.status, "publishing")); for (const r of toPublish) { try { const res = await publishRun(db, storage, r.id, { actor: { type: "system" } }); report.published.push(`${r.id}:${res.publicId}`); } catch (e) { report.errors.push({ runId: r.id, step: "publish", error: (e as Error).message }); log.error({ runId: r.id, err: (e as Error).message }, "publication failed; will retry"); } } } // stuck cancelling runs without lease for too long → cancelled try { const stuck = await db .select({ id: researchRuns.id }) .from(researchRuns) .where( and( eq(researchRuns.status, "cancelling"), lt(researchRuns.updatedAt, new Date(Date.now() - 3 * env.LEASE_TTL_SECONDS * 1000)), sql`not exists (select 1 from job_leases l where l.run_id = ${researchRuns.id})`, ), ); for (const r of stuck) await db.transaction((tx) => transition(tx, r.id, "cancelled", { actor: "scheduler", reason: "no lease held; cancellation finalized", }), ); } catch (e) { report.errors.push({ step: "finalizeCancel", error: (e as Error).message }); } return report; }