spb/autoscholar
Public
1import { createHash } from "node:crypto";2import { mkdir, readFile, stat, writeFile } from "node:fs/promises";3import { join } from "node:path";4import type { ArtifactKind, StageKey } from "@autoscholar/schemas";5import { listFiles } from "./latex.js";67export const MAX_ARTIFACT_BYTES = 256 * 1024 * 1024;89export function kindFor(path: string): ArtifactKind {10 if (path === "input/CLAUDE.md") return "instruction";11 if (path === "input/run.json") return "manifest";12 if (path.startsWith("plan/")) return "plan";13 if (path.startsWith("sources/")) return "source";14 if (path.startsWith("data/")) return "data";15 if (path.startsWith("code/")) return "code";16 if (path.startsWith("results/figures/")) return "figure";17 if (path.startsWith("results/tables/")) return "table";18 if (path.startsWith("results/")) return "result";19 if (path === "manuscript/main.pdf") return "pdf";20 if (path.startsWith("manuscript/")) return "manuscript";21 if (path.startsWith("review/")) return "review";22 if (path.startsWith("provenance/checksums")) return "checksum";23 if (path.startsWith("provenance/")) return "provenance";24 if (path.startsWith("logs/")) return "log";25 return "other";26}2728export function contentTypeFor(path: string): string {29 const ext = path.split(".").pop()?.toLowerCase();30 const map: Record<string, string> = {31 pdf: "application/pdf",32 md: "text/markdown; charset=utf-8",33 txt: "text/plain; charset=utf-8",34 json: "application/json",35 jsonl: "application/x-ndjson",36 tex: "text/x-tex; charset=utf-8",37 bib: "text/x-bibtex; charset=utf-8",38 cls: "text/x-tex; charset=utf-8",39 sty: "text/x-tex; charset=utf-8",40 bst: "text/plain; charset=utf-8",41 csv: "text/csv; charset=utf-8",42 png: "image/png",43 jpg: "image/jpeg",44 jpeg: "image/jpeg",45 svg: "image/svg+xml",46 py: "text/x-python; charset=utf-8",47 r: "text/plain; charset=utf-8",48 jl: "text/plain; charset=utf-8",49 sh: "text/x-shellscript; charset=utf-8",50 yaml: "text/yaml",51 yml: "text/yaml",52 toml: "text/plain; charset=utf-8",53 log: "text/plain; charset=utf-8",54 sha256: "text/plain; charset=utf-8",55 parquet: "application/vnd.apache.parquet",56 zip: "application/zip",57 lean: "text/plain; charset=utf-8",58 html: "text/html; charset=utf-8",59 ipynb: "application/x-ipynb+json",60 };61 return map[ext ?? ""] ?? "application/octet-stream";62}6364const SKIP = [65 /(^|\/)\.DS_Store$/,66 /\.(aux|toc|out|bbl|blg|fls|fdb_latexmk|synctex\.gz|xdv|nav|snm|lof|lot)$/,67 /(^|\/)\.pytest_cache\//,68 /\.pyc$/,69 /(^|\/)\.ipynb_checkpoints\//,70];7172export interface UploadRecord {73 path: string;74 sha256: string;75 bytes: number;76 stage: StageKey;77}7879export async function hashFile(p: string): Promise<{ sha256: string; bytes: number }> {80 const { createReadStream } = await import("node:fs");81 const h = createHash("sha256");82 let bytes = 0;83 for await (const chunk of createReadStream(p)) {84 h.update(chunk as Buffer);85 bytes += (chunk as Buffer).length;86 }87 return { sha256: h.digest("hex"), bytes };88}8990/** Uploads every new/changed file under the workspace (excluding platform dirs). Tracks state in .autoscholar/uploaded.json. */91export async function syncArtifacts(92 workspace: string,93 stage: StageKey,94 upload: (path: string, kind: ArtifactKind, contentType: string, buf: Buffer) => Promise<void>,95 emit: (kind: string, message: string, data?: Record<string, unknown>) => void,96 opts: { synthetic?: boolean } = {},97): Promise<UploadRecord[]> {98 const statePath = join(workspace, ".autoscholar/uploaded.json");99 let state: Record<string, string> = {};100 try {101 state = JSON.parse(await readFile(statePath, "utf8"));102 } catch {103 /* first upload */104 }105 const files = await listFiles(workspace);106 const uploaded: UploadRecord[] = [];107 let skipped = 0;108 for (const rel of files) {109 if (SKIP.some((r) => r.test(rel))) continue;110 if (111 rel.startsWith("home/") ||112 rel.startsWith(".autoscholar/") ||113 rel.startsWith(".claude/") ||114 rel === "CLAUDE.md"115 )116 continue;117 const abs = join(workspace, rel);118 const st = await stat(abs);119 if (st.size > MAX_ARTIFACT_BYTES) {120 skipped++;121 emit("artifact.skipped", `skipped ${rel} (${st.size} bytes > limit)`, { path: rel, bytes: st.size });122 continue;123 }124 const buf = await readFile(abs);125 const sha256 = createHash("sha256").update(buf).digest("hex");126 if (state[rel] === sha256) continue;127 await upload(rel, kindFor(rel), contentTypeFor(rel), buf);128 state[rel] = sha256;129 uploaded.push({ path: rel, sha256, bytes: buf.length, stage });130 }131 await mkdir(join(workspace, ".autoscholar"), { recursive: true });132 await writeFile(statePath, JSON.stringify(state, null, 1));133 if (uploaded.length || skipped)134 emit(135 "artifact.sync",136 `uploaded ${uploaded.length} artifact(s) after ${stage}${skipped ? `, ${skipped} skipped` : ""}`,137 { stage, uploaded: uploaded.length, skipped, synthetic: !!opts.synthetic },138 );139 return uploaded;140}141142/** Writes provenance/checksums.sha256 for every artifact file (excluding itself and platform dirs). */143export async function writeChecksums(workspace: string): Promise<string> {144 const files = (await listFiles(workspace)).filter(145 (f) =>146 !SKIP.some((r) => r.test(f)) &&147 !f.startsWith("home/") &&148 !f.startsWith(".autoscholar/") &&149 !f.startsWith(".claude/") &&150 f !== "CLAUDE.md" &&151 f !== "provenance/checksums.sha256",152 );153 const lines: string[] = [];154 const oversize: string[] = [];155 for (const f of files.sort()) {156 try {157 const st = await stat(join(workspace, f));158 if (st.size > MAX_ARTIFACT_BYTES) {159 oversize.push(`# not archived (oversize ${st.size} bytes): ${f}`);160 continue;161 }162 const { sha256 } = await hashFile(join(workspace, f));163 lines.push(`${sha256} ${f}`);164 } catch (e) {165 oversize.push(`# unreadable: ${f} (${(e as Error).message.slice(0, 80)})`);166 }167 }168 await mkdir(join(workspace, "provenance"), { recursive: true });169 const out = `${[...lines, ...oversize].join("\n")}\n`;170 await writeFile(join(workspace, "provenance/checksums.sha256"), out);171 return out;172}173