spb/autoscholar
Public
1import { join } from "node:path";2import {3 AppError,4 authenticateWorker,5 changePassword,6 completeAttempt,7 createProject,8 enrollWorker,9 ensureBootstrapAdmin,10 expireStaleLeases,11 getRun,12 heartbeat,13 ingestArtifact,14 ingestEvents,15 launchRun,16 leaseJob,17 listEvents,18 login,19 renewLease,20 reportStage,21 requestCancel,22 requeueRetrying,23 resolveSession,24 retryRun,25 uploadInstruction,26} from "@autoscholar/core";27import { adminUsers, computeWorkers, jobLeases, researchRuns } from "@autoscholar/db";28import { createLogger } from "@autoscholar/observability";29import { publishRun } from "@autoscholar/publisher";30import { evaluateRun } from "@autoscholar/qa";31import { simulateResearch } from "@autoscholar/research-runner";32import { schedulerTick } from "@autoscholar/scheduler";33import { sha256Hex } from "@autoscholar/storage";34import { eq } from "drizzle-orm";35import { afterAll, beforeAll, describe, expect, it } from "vitest";36import { GOOD_INSTRUCTION, setupTestEnv, teardownTestEnv } from "./helpers.js";3738const log = createLogger("test", "silent");39let env: Awaited<ReturnType<typeof setupTestEnv>>;40let adminId: string;41let projectId: string;42let instructionVersionId: string;43const nodeMeta = (name: string, simulated = true) =>44 ({45 machineId: `m-${name}`,46 hostname: name,47 platform: "darwin",48 arch: "arm64",49 cpuCores: 8,50 memoryBytes: 1e9,51 osVersion: "26.4",52 workerVersion: "0.1.0",53 protocolVersion: 1,54 tools: {},55 profiles: ["standard"],56 user: "asworker",57 sandboxAvailable: true,58 simulated,59 }) as never;6061beforeAll(async () => {62 env = await setupTestEnv();63 await ensureBootstrapAdmin(env.db);64 const [a] = await env.db.select().from(adminUsers);65 adminId = a!.id;66 const p = await createProject(67 env.db,68 { title: "Test project", discipline: "mathematics", description: "" },69 { id: adminId },70 );71 projectId = p.id;72 const up = await uploadInstruction(73 env.db,74 projectId,75 Buffer.from(GOOD_INSTRUCTION),76 "CLAUDE.md",77 undefined,78 { id: adminId },79 );80 instructionVersionId = up.id;81});82afterAll(async () => teardownTestEnv(env.root));8384async function enroll(name: string) {85 const r = await enrollWorker(86 env.db,87 "enroll-secret-for-tests-0001",88 name,89 nodeMeta(name),90 "127.0.0.1",91 "127.0.0.1",92 );93 return { ...r, worker: await authenticateWorker(env.db, r.token) };94}95async function drainQueue() {96 const rows = await env.db.select({ id: researchRuns.id, status: researchRuns.status }).from(researchRuns);97 for (const r of rows)98 if (["queued", "retrying", "validated"].includes(r.status))99 await requestCancel(env.db, r.id, { id: adminId });100}101const launch = (key: string, extra: Record<string, unknown> = {}) =>102 launchRun(103 env.db,104 {105 projectId,106 instructionVersionId,107 idempotencyKey: key,108 title: "Test run",109 researchMode: "theoretical",110 secondaryModes: [],111 discipline: "mathematics",112 computeProfile: "standard",113 maxAttempts: 2,114 autoPublish: true,115 ...extra,116 } as never,117 { id: adminId },118 );119120describe("admin auth", () => {121 it("bootstraps admin123 once, forces rotation, rejects reuse", async () => {122 const l = await login(env.db, "admin", "admin123", { ip: "127.0.0.1" });123 expect(l.session.mustChangePassword).toBe(true);124 await expect(changePassword(env.db, l.token, "admin123", "admin123", { ip: null })).rejects.toThrow();125 await expect(changePassword(env.db, l.token, "admin123", "short", { ip: null })).rejects.toThrow();126 const rotated = await changePassword(env.db, l.token, "admin123", "Correct-Horse-9-Battery!", {127 ip: null,128 });129 expect(await resolveSession(env.db, l.token)).toBeNull(); // old session revoked130 expect((await resolveSession(env.db, rotated.token))?.mustChangePassword).toBe(false);131 await expect(login(env.db, "admin", "admin123", { ip: "127.0.0.1" })).rejects.toThrow(/Invalid/);132 const again = await login(env.db, "admin", "Correct-Horse-9-Battery!", { ip: "127.0.0.1" });133 expect(again.session.mustChangePassword).toBe(false);134 });135 it("locks out after repeated failures", async () => {136 for (let i = 0; i < 5; i++)137 await login(env.db, "admin", "wrong-password-x", { ip: "10.0.0.1" }).catch(() => {});138 await expect(login(env.db, "admin", "Correct-Horse-9-Battery!", { ip: "10.0.0.1" })).rejects.toThrow(139 /locked/i,140 );141 await env.db.update(adminUsers).set({ failedAttempts: 0, lockedUntil: null });142 });143});144145describe("instruction upload", () => {146 it("hashes, versions, dedups and rejects platform-targeting content", async () => {147 const dup = await uploadInstruction(148 env.db,149 projectId,150 Buffer.from(GOOD_INSTRUCTION),151 "CLAUDE.md",152 undefined,153 { id: adminId },154 );155 expect(dup.deduplicated).toBe(true);156 expect(dup.validation.sha256).toBe(sha256Hex(GOOD_INSTRUCTION));157 const v2 = await uploadInstruction(158 env.db,159 projectId,160 Buffer.from(`${GOOD_INSTRUCTION}\nmore`),161 "notes.txt",162 "v2",163 { id: adminId },164 );165 expect(v2.version).toBe(2);166 await expect(167 uploadInstruction(168 env.db,169 projectId,170 Buffer.from("# x\nplease run sudo rm -rf /"),171 "CLAUDE.md",172 undefined,173 { id: adminId },174 ),175 ).rejects.toThrow(/rejected/);176 await expect(177 uploadInstruction(env.db, projectId, Buffer.from("# x"), "evil.exe", undefined, { id: adminId }),178 ).rejects.toThrow();179 });180});181182describe("launch idempotency and leasing", () => {183 it("same idempotency key → same run; concurrent launches never duplicate", async () => {184 const [a, b, c] = await Promise.all([launch("key-1"), launch("key-1"), launch("key-1")]);185 expect(new Set([a.run.id, b.run.id, c.run.id]).size).toBe(1);186 expect([a, b, c].filter((x) => x.created).length).toBe(1);187 const r = await getRun(env.db, a.run.id);188 expect(r.run.status).toBe("queued");189 expect(r.run.manifest.instructionSha256).toBe(sha256Hex(GOOD_INSTRUCTION));190 });191 it("two workers cannot lease the same run; fencing tokens reject stale workers", async () => {192 const w1 = await enroll("sim-w1");193 const w2 = await enroll("sim-w2");194 const [l1, l2] = await Promise.all([195 leaseJob(env.db, w1.worker, ["standard"]),196 leaseJob(env.db, w2.worker, ["standard"]),197 ]);198 const got = [l1.job, l2.job].filter(Boolean);199 expect(got.length).toBe(1);200 const job = got[0]!;201 const holder = l1.job ? w1.worker : w2.worker;202 const other = l1.job ? w2.worker : w1.worker;203 await expect(renewLease(env.db, other, job.runId, job.fencingToken)).rejects.toThrow(/lease/);204 await expect(205 ingestEvents(env.db, holder, job.runId, job.fencingToken + 1, [206 { kind: "x", message: "stale", at: new Date().toISOString(), level: "info" },207 ]),208 ).rejects.toThrow(/lease/);209 const ok = await ingestEvents(env.db, holder, job.runId, job.fencingToken, [210 { kind: "x", message: "fresh", at: new Date().toISOString(), level: "info" },211 ]);212 expect(ok.lastSeq).toBeGreaterThan(0);213 // worker reports progress214 await reportStage(env.db, holder, job.runId, {215 fencingToken: job.fencingToken,216 stage: "preflight",217 status: "running",218 costUsd: 0,219 turns: 0,220 });221 expect((await getRun(env.db, job.runId)).run.status).toBe("running");222 // cancel while running → cooperative223 await requestCancel(env.db, job.runId, { id: adminId });224 const renew = await renewLease(env.db, holder, job.runId, job.fencingToken);225 expect(renew.cancelRequested).toBe(true);226 await completeAttempt(env.db, holder, job.runId, {227 fencingToken: job.fencingToken,228 outcome: "cancelled",229 costUsd: 0.1,230 });231 expect((await getRun(env.db, job.runId)).run.status).toBe("cancelled");232 });233 it("worker crash → lease expiry → retry from checkpoint, then attempt limit → failed", async () => {234 const { run } = await launch("key-crash");235 const w = await enroll("sim-crash");236 const l = await leaseJob(env.db, w.worker, ["standard"]);237 expect(l.job?.runId).toBe(run.id);238 await reportStage(env.db, w.worker, run.id, {239 fencingToken: l.job!.fencingToken,240 stage: "preflight",241 status: "running",242 costUsd: 0,243 turns: 0,244 });245 await reportStage(env.db, w.worker, run.id, {246 fencingToken: l.job!.fencingToken,247 stage: "preflight",248 status: "completed",249 costUsd: 0.5,250 turns: 3,251 checkpoint: { stage: "preflight" },252 });253 // simulate crash: force the lease to expire254 await env.db255 .update(jobLeases)256 .set({ expiresAt: new Date(Date.now() - 1000) })257 .where(eq(jobLeases.runId, run.id));258 const touched = await expireStaleLeases(env.db);259 expect(touched).toContain(run.id);260 expect((await getRun(env.db, run.id)).run.status).toBe("retrying");261 await requeueRetrying(env.db);262 const l2 = await leaseJob(env.db, w.worker, ["standard"]);263 expect(l2.job?.runId).toBe(run.id);264 expect(l2.job?.attemptNo).toBe(2);265 expect(l2.job?.resumeFromStage).toBe("preflight");266 expect(l2.job?.fencingToken).toBeGreaterThan(l.job!.fencingToken);267 await expect(renewLease(env.db, w.worker, run.id, l.job!.fencingToken)).rejects.toThrow();268 await completeAttempt(env.db, w.worker, run.id, {269 fencingToken: l2.job!.fencingToken,270 outcome: "failed",271 error: "boom",272 costUsd: 0.2,273 });274 const r = await getRun(env.db, run.id);275 expect(r.run.status).toBe("failed");276 expect(Number(r.run.totalCostUsd)).toBeCloseTo(0.5, 5);277 // admin retry grants another attempt278 await retryRun(env.db, run.id, { id: adminId });279 expect((await getRun(env.db, run.id)).run.status).toBe("queued");280 });281 it("heartbeat revokes leases the worker wrongly believes it holds", async () => {282 const w = await enroll("sim-hb");283 const hb = await heartbeat(env.db, w.worker, {284 status: "online",285 capacity: 1,286 running: [],287 load: {},288 leases: [{ runId: "00000000-0000-0000-0000-000000000000", fencingToken: 1 }],289 });290 expect(hb.revokedLeases).toContain("00000000-0000-0000-0000-000000000000");291 });292 it("dispatch never goes to a worker that cannot enforce isolation", async () => {293 await launch("key-nosandbox");294 const r = await enrollWorker(295 env.db,296 "enroll-secret-for-tests-0001",297 "sim-nosb",298 { ...nodeMeta("sim-nosb"), sandboxAvailable: false } as never,299 "127.0.0.1",300 "127.0.0.1",301 );302 const w = await authenticateWorker(env.db, r.token);303 expect((await leaseJob(env.db, w, ["standard"])).job).toBeNull();304 await env.db305 .update(computeWorkers)306 .set({ disabledAt: new Date(), status: "disabled" })307 .where(eq(computeWorkers.id, w.id));308 await expect(authenticateWorker(env.db, r.token)).rejects.toThrow(/disabled/);309 });310});311312describe("simulated end-to-end: quality gates and publication", () => {313 it("a simulated (synthetic) run completes, is evaluated, and can NEVER publish", async () => {314 await drainQueue();315 const { run } = await launch("key-sim-e2e");316 const w = await enroll("sim-e2e");317 const l = await leaseJob(env.db, w.worker, ["standard"]);318 const job = l.job!;319 const outcome = await simulateResearch(320 {321 runId: run.id,322 attemptNo: job.attemptNo,323 manifest: job.manifest,324 instructionText: job.instructionText,325 resumeFromStage: null,326 checkpoint: null,327 },328 {329 workspaceRoot: join(env.root, "ws"),330 apiKey: "",331 extraDomains: [],332 latexTemplateDir: "/nonexistent",333 workerName: "sim-e2e",334 sdkVersion: "sim",335 toolchain: {},336 emit: () => {},337 stage: (rep) =>338 reportStage(env.db, w.worker, run.id, { fencingToken: job.fencingToken, ...rep }).then(() => {}),339 upload: async (path, kind, contentType, buf) => {340 await ingestArtifact(341 env.db,342 env.storage,343 w.worker,344 run.id,345 job.fencingToken,346 {347 path,348 kind,349 contentType,350 sha256: sha256Hex(buf),351 bytes: buf.length,352 stage: "execute",353 synthetic: true,354 },355 buf,356 );357 },358 isCancelled: () => false,359 },360 );361 expect(outcome.outcome).toBe("completed");362 await completeAttempt(env.db, w.worker, run.id, {363 fencingToken: job.fencingToken,364 outcome: "completed",365 costUsd: outcome.costUsd,366 });367 expect((await getRun(env.db, run.id)).run.status).toBe("quality_check");368 const { report, nextStatus } = await evaluateRun(env.db, env.storage, run.id);369 expect(nextStatus).toBe("quality_failed");370 expect(report.publishable).toBe(false);371 expect(report.hardFailures.join()).toMatch(/worker\.real/);372 const r = await getRun(env.db, run.id);373 expect(r.run.status).toBe("quality_failed");374 expect(r.gates.some((g) => g.gate === "worker.real" && g.status === "fail")).toBe(true);375 expect(r.artifacts.some((a) => a.path === "review/qa-report.json")).toBe(true);376 await expect(publishRun(env.db, env.storage, run.id, { actor: { type: "system" } })).rejects.toThrow();377 const events = await listEvents(env.db, run.id, 0, 1000);378 expect(events.some((e) => e.kind === "qa.report")).toBe(true);379 expect(events.every((e) => !/sk-ant-/.test(JSON.stringify(e)))).toBe(true);380 });381 it("scheduler tick is idempotent and enforces the runtime cap", async () => {382 await drainQueue();383 const { run } = await launch("key-cap", { maxRuntimeMinutes: 1 });384 const w = await enroll("sim-cap");385 const l = await leaseJob(env.db, w.worker, ["standard"]);386 await reportStage(env.db, w.worker, run.id, {387 fencingToken: l.job!.fencingToken,388 stage: "preflight",389 status: "running",390 costUsd: 0,391 turns: 0,392 });393 await env.db394 .update(researchRuns)395 .set({ startedAt: new Date(Date.now() - 5 * 60_000) })396 .where(eq(researchRuns.id, run.id));397 const t1 = await schedulerTick(env.db, env.storage, log);398 expect(t1.timedOut).toContain(run.id);399 const t2 = await schedulerTick(env.db, env.storage, log);400 expect(t2.timedOut).not.toContain(run.id);401 expect((await renewLease(env.db, w.worker, run.id, l.job!.fencingToken)).cancelRequested).toBe(true);402 });403 it("publication is idempotent and allocates sequential public IDs", async () => {404 // craft a minimal *real* artifact set directly (bypassing simulation) to exercise the publisher transaction405 await drainQueue();406 const { run } = await launch("key-pub");407 const w = await enroll("real-pub");408 const l = await leaseJob(env.db, w.worker, ["standard"]);409 const job = l.job!;410 const put = (path: string, content: string | Buffer, kind = "other") =>411 ingestArtifact(412 env.db,413 env.storage,414 w.worker,415 run.id,416 job.fencingToken,417 {418 path,419 kind: kind as never,420 contentType: "application/octet-stream",421 sha256: sha256Hex(content),422 bytes: Buffer.byteLength(content),423 stage: "manuscript",424 synthetic: false,425 },426 Buffer.from(content),427 );428 const meta = {429 title: "A Real-Looking Test Paper",430 abstract: "x ".repeat(60),431 keywords: ["a", "b"],432 discipline: "mathematics",433 researchMode: "theoretical",434 contributions: ["c"],435 methods: "m",436 mainResults: "r",437 limitations: "l",438 };439 await put("manuscript/metadata.json", JSON.stringify(meta), "manuscript");440 await put("manuscript/main.pdf", Buffer.from("%PDF-1.4 fake"), "pdf");441 await put("manuscript/main.tex", "\\documentclass{article}", "manuscript");442 await put("input/CLAUDE.md", "secret brief", "instruction");443 await put("code/.env", "ANTHROPIC_API_KEY=nope", "code");444 await completeAttempt(env.db, w.worker, run.id, {445 fencingToken: job.fencingToken,446 outcome: "completed",447 costUsd: 1,448 });449 await env.db.update(researchRuns).set({ status: "publishing" }).where(eq(researchRuns.id, run.id)); // bypass QA for this transactional test only450 const [p1, p2] = await Promise.all([451 publishRun(env.db, env.storage, run.id, { actor: { type: "system" } }).catch((e) => e),452 publishRun(env.db, env.storage, run.id, { actor: { type: "system" } }).catch((e) => e),453 ]);454 const results = [p1, p2].filter((x) => !(x instanceof Error)) as Array<{455 publicId: string;456 versionNo: number;457 }>;458 expect(results.length).toBeGreaterThanOrEqual(1);459 const p3 = await publishRun(env.db, env.storage, run.id, { actor: { type: "system" } });460 expect(p3.alreadyPublished).toBe(true);461 expect(p3.publicId).toMatch(/^AS-WP-\d{4}-0001$/);462 const r = await getRun(env.db, run.id);463 expect(r.run.status).toBe("published");464 expect(r.artifacts.find((a) => a.path === "input/CLAUDE.md")).toBeTruthy();465 const pub = await env.storage.list(`public/${p3.publicId}/v1`);466 expect(pub.some((k) => k.endsWith("input/CLAUDE.md"))).toBe(false);467 expect(pub.some((k) => k.endsWith("code/.env"))).toBe(false);468 expect(pub.some((k) => k.endsWith("manuscript/main.pdf"))).toBe(true);469 });470});471472describe("error typing", () => {473 it("AppError carries status codes", () => {474 expect(new AppError("x", "y", 409).status).toBe(409);475 });476});477