spb/autoscholar
Public
1import { join } from "node:path";2import {3 AppError,4 authenticateWorker,5 changePassword,6 completeAttempt,7 listRunArtifactsForWorker,8 readRunArtifactForWorker,9 createProject,10 enrollWorker,11 ensureBootstrapAdmin,12 expireStaleLeases,13 getRun,14 heartbeat,15 ingestArtifact,16 ingestEvents,17 launchRun,18 leaseJob,19 listEvents,20 login,21 renewLease,22 reportStage,23 requestCancel,24 requeueRetrying,25 resolveSession,26 retryRun,27 uploadInstruction,28} from "@autoscholar/core";29import { adminUsers, computeWorkers, jobLeases, researchRuns } from "@autoscholar/db";30import { createLogger } from "@autoscholar/observability";31import { publishRun } from "@autoscholar/publisher";32import { evaluateRun } from "@autoscholar/qa";33import { simulateResearch } from "@autoscholar/research-runner";34import { schedulerTick } from "@autoscholar/scheduler";35import { sha256Hex } from "@autoscholar/storage";36import { eq } from "drizzle-orm";37import { afterAll, beforeAll, describe, expect, it } from "vitest";38import { GOOD_INSTRUCTION, setupTestEnv, teardownTestEnv } from "./helpers.js";3940const log = createLogger("test", "silent");41let env: Awaited<ReturnType<typeof setupTestEnv>>;42let adminId: string;43let projectId: string;44let instructionVersionId: string;45const nodeMeta = (name: string, simulated = true) =>46 ({47 machineId: `m-${name}`,48 hostname: name,49 platform: "darwin",50 arch: "arm64",51 cpuCores: 8,52 memoryBytes: 1e9,53 osVersion: "26.4",54 workerVersion: "0.1.0",55 protocolVersion: 1,56 tools: {},57 profiles: ["standard"],58 user: "asworker",59 sandboxAvailable: true,60 simulated,61 }) as never;6263beforeAll(async () => {64 env = await setupTestEnv();65 await ensureBootstrapAdmin(env.db);66 const [a] = await env.db.select().from(adminUsers);67 adminId = a!.id;68 const p = await createProject(69 env.db,70 { title: "Test project", discipline: "mathematics", description: "" },71 { id: adminId },72 );73 projectId = p.id;74 const up = await uploadInstruction(75 env.db,76 projectId,77 Buffer.from(GOOD_INSTRUCTION),78 "CLAUDE.md",79 undefined,80 { id: adminId },81 );82 instructionVersionId = up.id;83});84afterAll(async () => teardownTestEnv(env.root));8586async function enroll(name: string) {87 const r = await enrollWorker(88 env.db,89 "enroll-secret-for-tests-0001",90 name,91 nodeMeta(name),92 "127.0.0.1",93 "127.0.0.1",94 );95 return { ...r, worker: await authenticateWorker(env.db, r.token) };96}97async function drainQueue() {98 const rows = await env.db.select({ id: researchRuns.id, status: researchRuns.status }).from(researchRuns);99 for (const r of rows)100 if (["queued", "retrying", "validated"].includes(r.status))101 await requestCancel(env.db, r.id, { id: adminId });102}103const launch = (key: string, extra: Record<string, unknown> = {}) =>104 launchRun(105 env.db,106 {107 projectId,108 instructionVersionId,109 idempotencyKey: key,110 title: "Test run",111 researchMode: "theoretical",112 secondaryModes: [],113 discipline: "mathematics",114 computeProfile: "standard",115 maxAttempts: 2,116 autoPublish: true,117 ...extra,118 } as never,119 { id: adminId },120 );121122describe("admin auth", () => {123 it("bootstraps admin123 once, forces rotation, rejects reuse", async () => {124 const l = await login(env.db, "admin", "admin123", { ip: "127.0.0.1" });125 expect(l.session.mustChangePassword).toBe(true);126 await expect(changePassword(env.db, l.token, "admin123", "admin123", { ip: null })).rejects.toThrow();127 await expect(changePassword(env.db, l.token, "admin123", "short", { ip: null })).rejects.toThrow();128 const rotated = await changePassword(env.db, l.token, "admin123", "Correct-Horse-9-Battery!", {129 ip: null,130 });131 expect(await resolveSession(env.db, l.token)).toBeNull(); // old session revoked132 expect((await resolveSession(env.db, rotated.token))?.mustChangePassword).toBe(false);133 await expect(login(env.db, "admin", "admin123", { ip: "127.0.0.1" })).rejects.toThrow(/Invalid/);134 const again = await login(env.db, "admin", "Correct-Horse-9-Battery!", { ip: "127.0.0.1" });135 expect(again.session.mustChangePassword).toBe(false);136 });137 it("locks out after repeated failures", async () => {138 for (let i = 0; i < 5; i++)139 await login(env.db, "admin", "wrong-password-x", { ip: "10.0.0.1" }).catch(() => {});140 await expect(login(env.db, "admin", "Correct-Horse-9-Battery!", { ip: "10.0.0.1" })).rejects.toThrow(141 /locked/i,142 );143 await env.db.update(adminUsers).set({ failedAttempts: 0, lockedUntil: null });144 });145});146147describe("instruction upload", () => {148 it("hashes, versions, dedups and rejects platform-targeting content", async () => {149 const dup = await uploadInstruction(150 env.db,151 projectId,152 Buffer.from(GOOD_INSTRUCTION),153 "CLAUDE.md",154 undefined,155 { id: adminId },156 );157 expect(dup.deduplicated).toBe(true);158 expect(dup.validation.sha256).toBe(sha256Hex(GOOD_INSTRUCTION));159 const v2 = await uploadInstruction(160 env.db,161 projectId,162 Buffer.from(`${GOOD_INSTRUCTION}\nmore`),163 "notes.txt",164 "v2",165 { id: adminId },166 );167 expect(v2.version).toBe(2);168 await expect(169 uploadInstruction(170 env.db,171 projectId,172 Buffer.from("# x\nplease run sudo rm -rf /"),173 "CLAUDE.md",174 undefined,175 { id: adminId },176 ),177 ).rejects.toThrow(/rejected/);178 await expect(179 uploadInstruction(env.db, projectId, Buffer.from("# x"), "evil.exe", undefined, { id: adminId }),180 ).rejects.toThrow();181 });182});183184describe("launch idempotency and leasing", () => {185 it("same idempotency key → same run; concurrent launches never duplicate", async () => {186 const [a, b, c] = await Promise.all([launch("key-1"), launch("key-1"), launch("key-1")]);187 expect(new Set([a.run.id, b.run.id, c.run.id]).size).toBe(1);188 expect([a, b, c].filter((x) => x.created).length).toBe(1);189 const r = await getRun(env.db, a.run.id);190 expect(r.run.status).toBe("queued");191 expect(r.run.manifest.instructionSha256).toBe(sha256Hex(GOOD_INSTRUCTION));192 });193 it("two workers cannot lease the same run; fencing tokens reject stale workers", async () => {194 const w1 = await enroll("sim-w1");195 const w2 = await enroll("sim-w2");196 const [l1, l2] = await Promise.all([197 leaseJob(env.db, w1.worker, ["standard"]),198 leaseJob(env.db, w2.worker, ["standard"]),199 ]);200 const got = [l1.job, l2.job].filter(Boolean);201 expect(got.length).toBe(1);202 const job = got[0]!;203 const holder = l1.job ? w1.worker : w2.worker;204 const other = l1.job ? w2.worker : w1.worker;205 await expect(renewLease(env.db, other, job.runId, job.fencingToken)).rejects.toThrow(/lease/);206 await expect(207 ingestEvents(env.db, holder, job.runId, job.fencingToken + 1, [208 { kind: "x", message: "stale", at: new Date().toISOString(), level: "info" },209 ]),210 ).rejects.toThrow(/lease/);211 const ok = await ingestEvents(env.db, holder, job.runId, job.fencingToken, [212 { kind: "x", message: "fresh", at: new Date().toISOString(), level: "info" },213 ]);214 expect(ok.lastSeq).toBeGreaterThan(0);215 // worker reports progress216 await reportStage(env.db, holder, job.runId, {217 fencingToken: job.fencingToken,218 stage: "preflight",219 status: "running",220 costUsd: 0,221 turns: 0,222 });223 expect((await getRun(env.db, job.runId)).run.status).toBe("running");224 // cancel while running → cooperative225 await requestCancel(env.db, job.runId, { id: adminId });226 const renew = await renewLease(env.db, holder, job.runId, job.fencingToken);227 expect(renew.cancelRequested).toBe(true);228 await completeAttempt(env.db, holder, job.runId, {229 fencingToken: job.fencingToken,230 outcome: "cancelled",231 costUsd: 0.1,232 });233 expect((await getRun(env.db, job.runId)).run.status).toBe("cancelled");234 });235 it("worker crash → lease expiry → retry from checkpoint, then attempt limit → failed", async () => {236 const { run } = await launch("key-crash");237 const w = await enroll("sim-crash");238 const l = await leaseJob(env.db, w.worker, ["standard"]);239 expect(l.job?.runId).toBe(run.id);240 await reportStage(env.db, w.worker, run.id, {241 fencingToken: l.job!.fencingToken,242 stage: "preflight",243 status: "running",244 costUsd: 0,245 turns: 0,246 });247 await reportStage(env.db, w.worker, run.id, {248 fencingToken: l.job!.fencingToken,249 stage: "preflight",250 status: "completed",251 costUsd: 0.5,252 turns: 3,253 checkpoint: { stage: "preflight" },254 });255 // simulate crash: force the lease to expire256 await env.db257 .update(jobLeases)258 .set({ expiresAt: new Date(Date.now() - 1000) })259 .where(eq(jobLeases.runId, run.id));260 const touched = await expireStaleLeases(env.db);261 expect(touched).toContain(run.id);262 expect((await getRun(env.db, run.id)).run.status).toBe("retrying");263 await requeueRetrying(env.db);264 const l2 = await leaseJob(env.db, w.worker, ["standard"]);265 expect(l2.job?.runId).toBe(run.id);266 expect(l2.job?.attemptNo).toBe(2);267 expect(l2.job?.resumeFromStage).toBe("preflight");268 expect(l2.job?.fencingToken).toBeGreaterThan(l.job!.fencingToken);269 await expect(renewLease(env.db, w.worker, run.id, l.job!.fencingToken)).rejects.toThrow();270 await completeAttempt(env.db, w.worker, run.id, {271 fencingToken: l2.job!.fencingToken,272 outcome: "failed",273 error: "boom",274 costUsd: 0.2,275 });276 const r = await getRun(env.db, run.id);277 expect(r.run.status).toBe("failed");278 expect(Number(r.run.totalCostUsd)).toBeCloseTo(0.5, 5);279 // admin retry grants another attempt280 await retryRun(env.db, run.id, { id: adminId });281 expect((await getRun(env.db, run.id)).run.status).toBe("queued");282 });283 it("heartbeat revokes leases the worker wrongly believes it holds", async () => {284 const w = await enroll("sim-hb");285 const hb = await heartbeat(env.db, w.worker, {286 status: "online",287 capacity: 1,288 running: [],289 load: {},290 leases: [{ runId: "00000000-0000-0000-0000-000000000000", fencingToken: 1 }],291 });292 expect(hb.revokedLeases).toContain("00000000-0000-0000-0000-000000000000");293 });294 it("dispatch never goes to a worker that cannot enforce isolation", async () => {295 await launch("key-nosandbox");296 const r = await enrollWorker(297 env.db,298 "enroll-secret-for-tests-0001",299 "sim-nosb",300 { ...nodeMeta("sim-nosb"), sandboxAvailable: false } as never,301 "127.0.0.1",302 "127.0.0.1",303 );304 const w = await authenticateWorker(env.db, r.token);305 expect((await leaseJob(env.db, w, ["standard"])).job).toBeNull();306 await env.db307 .update(computeWorkers)308 .set({ disabledAt: new Date(), status: "disabled" })309 .where(eq(computeWorkers.id, w.id));310 await expect(authenticateWorker(env.db, r.token)).rejects.toThrow(/disabled/);311 });312});313314describe("simulated end-to-end: quality gates and publication", () => {315 it("a simulated (synthetic) run completes, is evaluated, and can NEVER publish", async () => {316 await drainQueue();317 const { run } = await launch("key-sim-e2e");318 const w = await enroll("sim-e2e");319 const l = await leaseJob(env.db, w.worker, ["standard"]);320 const job = l.job!;321 const outcome = await simulateResearch(322 {323 runId: run.id,324 attemptNo: job.attemptNo,325 manifest: job.manifest,326 instructionText: job.instructionText,327 resumeFromStage: null,328 checkpoint: null,329 },330 {331 workspaceRoot: join(env.root, "ws"),332 apiKey: "",333 extraDomains: [],334 latexTemplateDir: "/nonexistent",335 workerName: "sim-e2e",336 sdkVersion: "sim",337 toolchain: {},338 emit: () => {},339 stage: (rep) =>340 reportStage(env.db, w.worker, run.id, { fencingToken: job.fencingToken, ...rep }).then(() => {}),341 upload: async (path, kind, contentType, buf) => {342 await ingestArtifact(343 env.db,344 env.storage,345 w.worker,346 run.id,347 job.fencingToken,348 {349 path,350 kind,351 contentType,352 sha256: sha256Hex(buf),353 bytes: buf.length,354 stage: "execute",355 synthetic: true,356 },357 buf,358 );359 },360 isCancelled: () => false,361 },362 );363 expect(outcome.outcome).toBe("completed");364 await completeAttempt(env.db, w.worker, run.id, {365 fencingToken: job.fencingToken,366 outcome: "completed",367 costUsd: outcome.costUsd,368 });369 expect((await getRun(env.db, run.id)).run.status).toBe("quality_check");370 const { report, nextStatus } = await evaluateRun(env.db, env.storage, run.id);371 expect(nextStatus).toBe("quality_failed");372 expect(report.publishable).toBe(false);373 expect(report.hardFailures.join()).toMatch(/worker\.real/);374 const r = await getRun(env.db, run.id);375 expect(r.run.status).toBe("quality_failed");376 expect(r.gates.some((g) => g.gate === "worker.real" && g.status === "fail")).toBe(true);377 expect(r.artifacts.some((a) => a.path === "review/qa-report.json")).toBe(true);378 await expect(publishRun(env.db, env.storage, run.id, { actor: { type: "system" } })).rejects.toThrow();379 const events = await listEvents(env.db, run.id, 0, 1000);380 expect(events.some((e) => e.kind === "qa.report")).toBe(true);381 expect(events.every((e) => !/sk-ant-/.test(JSON.stringify(e)))).toBe(true);382 });383 it("scheduler tick is idempotent and enforces the runtime cap", async () => {384 await drainQueue();385 const { run } = await launch("key-cap", { maxRuntimeMinutes: 1 });386 const w = await enroll("sim-cap");387 const l = await leaseJob(env.db, w.worker, ["standard"]);388 await reportStage(env.db, w.worker, run.id, {389 fencingToken: l.job!.fencingToken,390 stage: "preflight",391 status: "running",392 costUsd: 0,393 turns: 0,394 });395 await env.db396 .update(researchRuns)397 .set({ startedAt: new Date(Date.now() - 5 * 60_000) })398 .where(eq(researchRuns.id, run.id));399 const t1 = await schedulerTick(env.db, env.storage, log);400 expect(t1.timedOut).toContain(run.id);401 const t2 = await schedulerTick(env.db, env.storage, log);402 expect(t2.timedOut).not.toContain(run.id);403 expect((await renewLease(env.db, w.worker, run.id, l.job!.fencingToken)).cancelRequested).toBe(true);404 });405 it("publication is idempotent and allocates sequential public IDs", async () => {406 // craft a minimal *real* artifact set directly (bypassing simulation) to exercise the publisher transaction407 await drainQueue();408 const { run } = await launch("key-pub");409 const w = await enroll("real-pub");410 const l = await leaseJob(env.db, w.worker, ["standard"]);411 const job = l.job!;412 const put = (path: string, content: string | Buffer, kind = "other") =>413 ingestArtifact(414 env.db,415 env.storage,416 w.worker,417 run.id,418 job.fencingToken,419 {420 path,421 kind: kind as never,422 contentType: "application/octet-stream",423 sha256: sha256Hex(content),424 bytes: Buffer.byteLength(content),425 stage: "manuscript",426 synthetic: false,427 },428 Buffer.from(content),429 );430 const meta = {431 title: "A Real-Looking Test Paper",432 abstract: "x ".repeat(60),433 keywords: ["a", "b"],434 discipline: "mathematics",435 researchMode: "theoretical",436 contributions: ["c"],437 methods: "m",438 mainResults: "r",439 limitations: "l",440 };441 await put("manuscript/metadata.json", JSON.stringify(meta), "manuscript");442 await put("manuscript/main.pdf", Buffer.from("%PDF-1.4 fake"), "pdf");443 await put("manuscript/main.tex", "\\documentclass{article}", "manuscript");444 await put("input/CLAUDE.md", "secret brief", "instruction");445 await put("code/.env", "ANTHROPIC_API_KEY=nope", "code");446 await completeAttempt(env.db, w.worker, run.id, {447 fencingToken: job.fencingToken,448 outcome: "completed",449 costUsd: 1,450 });451 await env.db.update(researchRuns).set({ status: "publishing" }).where(eq(researchRuns.id, run.id)); // bypass QA for this transactional test only452 const [p1, p2] = await Promise.all([453 publishRun(env.db, env.storage, run.id, { actor: { type: "system" } }).catch((e) => e),454 publishRun(env.db, env.storage, run.id, { actor: { type: "system" } }).catch((e) => e),455 ]);456 const results = [p1, p2].filter((x) => !(x instanceof Error)) as Array<{457 publicId: string;458 versionNo: number;459 }>;460 expect(results.length).toBeGreaterThanOrEqual(1);461 const p3 = await publishRun(env.db, env.storage, run.id, { actor: { type: "system" } });462 expect(p3.alreadyPublished).toBe(true);463 expect(p3.publicId).toMatch(/^AS-WP-\d{4}-\d{4}$/);464 expect(p3.publicId).toBe((run.manifest as { paperNumber?: string }).paperNumber);465 const r = await getRun(env.db, run.id);466 expect(r.run.status).toBe("published");467 expect(r.artifacts.find((a) => a.path === "input/CLAUDE.md")).toBeTruthy();468 const pub = await env.storage.list(`public/${p3.publicId}/v1`);469 expect(pub.some((k) => k.endsWith("input/CLAUDE.md"))).toBe(false);470 expect(pub.some((k) => k.endsWith("code/.env"))).toBe(false);471 expect(pub.some((k) => k.endsWith("manuscript/main.pdf"))).toBe(true);472 });473});474475describe("resume on another node", () => {476 it("lets only the lease holder list and download the latest artifacts", async () => {477 await drainQueue();478 const { run } = await launch("key-resume");479 const w1 = await enroll("sim-r1");480 const l1 = await leaseJob(env.db, w1.worker, ["standard"]);481 const job = l1.job!;482 const put = (path: string, content: string) => ingestArtifact(env.db, env.storage, w1.worker, run.id, job.fencingToken, { path, kind: "plan", contentType: "text/markdown", sha256: sha256Hex(content), bytes: Buffer.byteLength(content), stage: "preflight", synthetic: false }, Buffer.from(content));483 await put("plan/research-plan.md", "v1");484 await put("plan/research-plan.md", "v2 changed");485 await put("plan/feasibility.json", "{}");486 const list = await listRunArtifactsForWorker(env.db, w1.worker, run.id, job.fencingToken);487 expect(list.map((a) => `${a.path}@${a.version}`).sort()).toEqual(["plan/feasibility.json@1", "plan/research-plan.md@2"]);488 const { buf } = await readRunArtifactForWorker(env.db, env.storage, w1.worker, run.id, job.fencingToken, "plan/research-plan.md");489 expect(buf.toString()).toBe("v2 changed");490 const w2 = await enroll("sim-r2");491 await expect(listRunArtifactsForWorker(env.db, w2.worker, run.id, job.fencingToken)).rejects.toThrow(/lease/);492 await completeAttempt(env.db, w1.worker, run.id, { fencingToken: job.fencingToken, outcome: "cancelled", costUsd: 0 });493 });494});495496describe("error typing", () => {497 it("AppError carries status codes", () => {498 expect(new AppError("x", "y", 409).status).toBe(409);499 });500});501