import { createHash } from "node:crypto"; import { mkdir, readFile, stat, writeFile } from "node:fs/promises"; import { join } from "node:path"; import type { ArtifactKind, StageKey } from "@autoscholar/schemas"; import { listFiles } from "./latex.js"; export const MAX_ARTIFACT_BYTES = 256 * 1024 * 1024; export function kindFor(path: string): ArtifactKind { if (path === "input/CLAUDE.md") return "instruction"; if (path === "input/run.json") return "manifest"; if (path.startsWith("plan/")) return "plan"; if (path.startsWith("sources/")) return "source"; if (path.startsWith("data/")) return "data"; if (path.startsWith("code/")) return "code"; if (path.startsWith("results/figures/")) return "figure"; if (path.startsWith("results/tables/")) return "table"; if (path.startsWith("results/")) return "result"; if (path === "manuscript/main.pdf") return "pdf"; if (path.startsWith("manuscript/")) return "manuscript"; if (path.startsWith("review/")) return "review"; if (path.startsWith("provenance/checksums")) return "checksum"; if (path.startsWith("provenance/")) return "provenance"; if (path.startsWith("logs/")) return "log"; return "other"; } export function contentTypeFor(path: string): string { const ext = path.split(".").pop()?.toLowerCase(); const map: Record = { pdf: "application/pdf", md: "text/markdown; charset=utf-8", txt: "text/plain; charset=utf-8", json: "application/json", jsonl: "application/x-ndjson", tex: "text/x-tex; charset=utf-8", bib: "text/x-bibtex; charset=utf-8", cls: "text/x-tex; charset=utf-8", sty: "text/x-tex; charset=utf-8", bst: "text/plain; charset=utf-8", csv: "text/csv; charset=utf-8", png: "image/png", jpg: "image/jpeg", jpeg: "image/jpeg", svg: "image/svg+xml", py: "text/x-python; charset=utf-8", r: "text/plain; charset=utf-8", jl: "text/plain; charset=utf-8", sh: "text/x-shellscript; charset=utf-8", yaml: "text/yaml", yml: "text/yaml", toml: "text/plain; charset=utf-8", log: "text/plain; charset=utf-8", sha256: "text/plain; charset=utf-8", parquet: "application/vnd.apache.parquet", zip: "application/zip", lean: "text/plain; charset=utf-8", html: "text/html; charset=utf-8", ipynb: "application/x-ipynb+json", }; return map[ext ?? ""] ?? "application/octet-stream"; } const SKIP = [ /(^|\/)\.DS_Store$/, /\.(aux|toc|out|bbl|blg|fls|fdb_latexmk|synctex\.gz|xdv|nav|snm|lof|lot)$/, /(^|\/)\.pytest_cache\//, /\.pyc$/, /(^|\/)\.ipynb_checkpoints\//, ]; export interface UploadRecord { path: string; sha256: string; bytes: number; stage: StageKey; } export async function hashFile(p: string): Promise<{ sha256: string; bytes: number }> { const buf = await readFile(p); return { sha256: createHash("sha256").update(buf).digest("hex"), bytes: buf.length }; } /** Uploads every new/changed file under the workspace (excluding platform dirs). Tracks state in .autoscholar/uploaded.json. */ export async function syncArtifacts( workspace: string, stage: StageKey, upload: (path: string, kind: ArtifactKind, contentType: string, buf: Buffer) => Promise, emit: (kind: string, message: string, data?: Record) => void, opts: { synthetic?: boolean } = {}, ): Promise { const statePath = join(workspace, ".autoscholar/uploaded.json"); let state: Record = {}; try { state = JSON.parse(await readFile(statePath, "utf8")); } catch { /* first upload */ } const files = await listFiles(workspace); const uploaded: UploadRecord[] = []; let skipped = 0; for (const rel of files) { if (SKIP.some((r) => r.test(rel))) continue; if ( rel.startsWith("home/") || rel.startsWith(".autoscholar/") || rel.startsWith(".claude/") || rel === "CLAUDE.md" ) continue; const abs = join(workspace, rel); const st = await stat(abs); if (st.size > MAX_ARTIFACT_BYTES) { skipped++; emit("artifact.skipped", `skipped ${rel} (${st.size} bytes > limit)`, { path: rel, bytes: st.size }); continue; } const buf = await readFile(abs); const sha256 = createHash("sha256").update(buf).digest("hex"); if (state[rel] === sha256) continue; await upload(rel, kindFor(rel), contentTypeFor(rel), buf); state[rel] = sha256; uploaded.push({ path: rel, sha256, bytes: buf.length, stage }); } await mkdir(join(workspace, ".autoscholar"), { recursive: true }); await writeFile(statePath, JSON.stringify(state, null, 1)); if (uploaded.length || skipped) emit( "artifact.sync", `uploaded ${uploaded.length} artifact(s) after ${stage}${skipped ? `, ${skipped} skipped` : ""}`, { stage, uploaded: uploaded.length, skipped, synthetic: !!opts.synthetic }, ); return uploaded; } /** Writes provenance/checksums.sha256 for every artifact file (excluding itself and platform dirs). */ export async function writeChecksums(workspace: string): Promise { const files = (await listFiles(workspace)).filter( (f) => !SKIP.some((r) => r.test(f)) && !f.startsWith("home/") && !f.startsWith(".autoscholar/") && !f.startsWith(".claude/") && f !== "CLAUDE.md" && f !== "provenance/checksums.sha256", ); const lines: string[] = []; for (const f of files.sort()) { const { sha256 } = await hashFile(join(workspace, f)); lines.push(`${sha256} ${f}`); } await mkdir(join(workspace, "provenance"), { recursive: true }); const out = `${lines.join("\n")}\n`; await writeFile(join(workspace, "provenance/checksums.sha256"), out); return out; }