spb/autoscholar
Public
1import { randomUUID } from "node:crypto";2import { appendEvent, getEnv, getSetting, SETTING_KEYS, transition } from "@autoscholar/core";3import {4 artifacts,5 computeWorkers,6 type Db,7 publicationDecisions,8 qualityGateResults,9 researchRuns,10 reviewFindings,11 runAttempts,12} from "@autoscholar/db";13import { effectiveAutoPublish, MODE_PROFILES } from "@autoscholar/research-contract";14import { type GateResult, type QaReport, type ResearchMode, ReviewReport } from "@autoscholar/schemas";15import type { FilesystemStorage } from "@autoscholar/storage";16import { and, desc, eq } from "drizzle-orm";17import { type ArtifactView, runGates } from "./gates.js";1819export async function loadArtifactViews(20 db: Db,21 storage: FilesystemStorage,22 runId: string,23): Promise<ArtifactView[]> {24 const rows = await db25 .select()26 .from(artifacts)27 .where(eq(artifacts.runId, runId))28 .orderBy(desc(artifacts.version));29 const latest = new Map<string, (typeof rows)[number]>();30 for (const r of rows) if (!latest.has(r.path)) latest.set(r.path, r);31 return [...latest.values()].map((r) => ({32 path: r.path,33 kind: r.kind,34 sha256: r.sha256,35 bytes: r.bytes,36 synthetic: r.synthetic,37 read: () => storage.get(r.storageKey),38 storedSha256: async () => (await storage.head(r.storageKey))?.sha256 ?? null,39 }));40}4142/** Stage 9 — deterministic quality decision computed on the control plane. Never from model text. */43export async function evaluateRun(44 db: Db,45 storage: FilesystemStorage,46 runId: string,47): Promise<{ report: QaReport; nextStatus: string }> {48 const env = getEnv();49 const [run] = await db.select().from(researchRuns).where(eq(researchRuns.id, runId)).limit(1);50 if (!run) throw new Error("run not found");51 const [attempt] = await db52 .select()53 .from(runAttempts)54 .where(and(eq(runAttempts.runId, runId), eq(runAttempts.attemptNo, run.currentAttemptNo)))55 .limit(1);56 const [worker] = attempt?.workerId57 ? await db.select().from(computeWorkers).where(eq(computeWorkers.id, attempt.workerId)).limit(1)58 : [];59 const nodeMeta = (worker?.nodeMeta ?? {}) as Record<string, unknown>;60 const views = await loadArtifactViews(db, storage, runId);61 const mode = run.researchMode as ResearchMode;62 // persist reviewer findings for the admin quality page63 const rrView = views.find((v) => v.path === "review/review-1.json");64 if (rrView) {65 try {66 const rr = ReviewReport.parse(JSON.parse((await rrView.read()).toString("utf8")));67 for (const f of rr.findings) {68 await db69 .insert(reviewFindings)70 .values({71 runId,72 round: rr.round,73 findingKey: f.id,74 severity: f.severity,75 category: f.category,76 title: f.title,77 detail: f.detail,78 proposedTest: f.proposedTest ?? null,79 resolution: f.resolution,80 resolutionNote: f.resolutionNote ?? null,81 })82 .onConflictDoUpdate({83 target: [reviewFindings.runId, reviewFindings.round, reviewFindings.findingKey],84 set: { resolution: f.resolution, resolutionNote: f.resolutionNote ?? null, detail: f.detail },85 });86 }87 } catch {88 /* gate will report */89 }90 }91 const gates = await runGates({92 mode,93 secondaryModes: run.secondaryModes,94 artifacts: views,95 attemptOutcome: attempt?.status ?? null,96 cancelRequested: run.cancelRequested,97 workerSimulated: nodeMeta.simulated === true || (worker?.name ?? "").startsWith("sim-"),98 reviewFindingsOpenBlockers: 0,99 });100 const hardFailures = gates101 .filter((g) => g.severity === "hard" && g.status === "fail")102 .map((g) => `${g.gate}: ${g.message}`);103 const warnings = gates.filter((g) => g.status === "warn").map((g) => `${g.gate}: ${g.message}`);104 const publishable = hardFailures.length === 0;105 const evaluationId = randomUUID();106 const report: QaReport = {107 policyVersion: run.qualityPolicyVersion,108 mode,109 evaluatedAt: new Date().toISOString(),110 gates,111 publishable,112 hardFailures,113 warnings,114 synthetic: views.some((v) => v.synthetic),115 };116 await db.transaction(async (tx) => {117 for (const g of gates)118 await tx.insert(qualityGateResults).values({119 runId,120 evaluationId,121 policyVersion: run.qualityPolicyVersion,122 mode,123 gate: g.gate,124 severity: g.severity,125 status: g.status,126 message: g.message,127 evidence: g.evidence,128 });129 });130 // store the report as a run artifact (review/qa-report.json) produced by the control plane131 const key = `runs/${runId}/a${run.currentAttemptNo}/review/qa-report.json`;132 const buf = Buffer.from(JSON.stringify(report, null, 2));133 const stored = await storage.put(key, buf);134 const [prevQa] = await db135 .select({ version: artifacts.version, sha256: artifacts.sha256 })136 .from(artifacts)137 .where(and(eq(artifacts.runId, runId), eq(artifacts.path, "review/qa-report.json")))138 .orderBy(desc(artifacts.version))139 .limit(1);140 if (!prevQa || prevQa.sha256 !== stored.sha256) {141 const version = (prevQa?.version ?? 0) + 1;142 const vkey = version === 1 ? key : `${key}.v${version}`;143 if (version > 1) await storage.put(vkey, buf);144 await db.insert(artifacts).values({145 runId,146 attemptId: attempt?.id ?? null,147 stage: "quality",148 path: "review/qa-report.json",149 version,150 kind: "review",151 sha256: stored.sha256,152 bytes: stored.bytes,153 contentType: "application/json",154 storageKey: vkey,155 synthetic: false,156 });157 }158 let nextStatus: string;159 await db.transaction(async (tx) => {160 if (!publishable) {161 await tx.insert(publicationDecisions).values({162 runId,163 evaluationId,164 decision: "reject",165 decidedBy: "system",166 rationale: `hard gates failed: ${hardFailures.join(" | ")}`.slice(0, 4000),167 gateSummary: { hard: hardFailures.length, warnings: warnings.length },168 });169 await transition(tx, runId, "quality_failed", {170 actor: "qa",171 reason: `${hardFailures.length} hard gate(s) failed`,172 data: { evaluationId, hardFailures: hardFailures.slice(0, 10) },173 });174 nextStatus = "quality_failed";175 return;176 }177 await transition(tx, runId, "ready_to_publish", {178 actor: "qa",179 reason: "all hard gates passed",180 data: { evaluationId, warnings: warnings.length },181 });182 const halted =183 env.PUBLISHING_ENABLED !== 1 ||184 (await getSetting(tx as unknown as Db, SETTING_KEYS.publishingHalted, false));185 const haltedModes = await getSetting<string[]>(186 tx as unknown as Db,187 SETTING_KEYS.publishingHaltedModes,188 [],189 );190 const auto = effectiveAutoPublish(mode, run.discipline, run.autoPublish);191 if (auto.allowed && !halted && !haltedModes.includes(mode) && MODE_PROFILES[mode].autoPublishAllowed) {192 await tx.insert(publicationDecisions).values({193 runId,194 evaluationId,195 decision: "publish",196 decidedBy: "system",197 rationale: "all hard gates passed; autoPublish=true",198 gateSummary: { warnings },199 });200 await transition(tx, runId, "publishing", { actor: "qa", reason: "auto-publish policy" });201 nextStatus = "publishing";202 } else {203 await tx.insert(publicationDecisions).values({204 runId,205 evaluationId,206 decision: "hold",207 decidedBy: "system",208 rationale: halted209 ? "publication halted by founder control"210 : (auto.reason ?? "autoPublish=false; awaiting admin approval"),211 gateSummary: { warnings },212 });213 await transition(tx, runId, "awaiting_approval", {214 actor: "qa",215 reason: halted ? "publishing halted" : (auto.reason ?? "manual approval required"),216 });217 nextStatus = "awaiting_approval";218 }219 });220 await db.transaction((tx) =>221 appendEvent(tx, runId, {222 kind: "qa.report",223 message: publishable224 ? `QA passed (${warnings.length} warnings)`225 : `QA failed: ${hardFailures.length} hard gate(s)`,226 data: { evaluationId },227 at: new Date().toISOString(),228 }),229 );230 return { report, nextStatus: nextStatus! };231}232233export type { GateResult };234