spb/autoscholar

1import { createHash } from "node:crypto";2import { mkdir, readFile, stat, writeFile } from "node:fs/promises";3import { join } from "node:path";4import type { ArtifactKind, StageKey } from "@autoscholar/schemas";5import { listFiles } from "./latex.js";67export const MAX_ARTIFACT_BYTES = 256 * 1024 * 1024;89export function kindFor(path: string): ArtifactKind {10  if (path === "input/CLAUDE.md") return "instruction";11  if (path === "input/run.json") return "manifest";12  if (path.startsWith("plan/")) return "plan";13  if (path.startsWith("sources/")) return "source";14  if (path.startsWith("data/")) return "data";15  if (path.startsWith("code/")) return "code";16  if (path.startsWith("results/figures/")) return "figure";17  if (path.startsWith("results/tables/")) return "table";18  if (path.startsWith("results/")) return "result";19  if (path === "manuscript/main.pdf") return "pdf";20  if (path.startsWith("manuscript/")) return "manuscript";21  if (path.startsWith("review/")) return "review";22  if (path.startsWith("provenance/checksums")) return "checksum";23  if (path.startsWith("provenance/")) return "provenance";24  if (path.startsWith("logs/")) return "log";25  return "other";26}2728export function contentTypeFor(path: string): string {29  const ext = path.split(".").pop()?.toLowerCase();30  const map: Record<string, string> = {31    pdf: "application/pdf",32    md: "text/markdown; charset=utf-8",33    txt: "text/plain; charset=utf-8",34    json: "application/json",35    jsonl: "application/x-ndjson",36    tex: "text/x-tex; charset=utf-8",37    bib: "text/x-bibtex; charset=utf-8",38    cls: "text/x-tex; charset=utf-8",39    sty: "text/x-tex; charset=utf-8",40    bst: "text/plain; charset=utf-8",41    csv: "text/csv; charset=utf-8",42    png: "image/png",43    jpg: "image/jpeg",44    jpeg: "image/jpeg",45    svg: "image/svg+xml",46    py: "text/x-python; charset=utf-8",47    r: "text/plain; charset=utf-8",48    jl: "text/plain; charset=utf-8",49    sh: "text/x-shellscript; charset=utf-8",50    yaml: "text/yaml",51    yml: "text/yaml",52    toml: "text/plain; charset=utf-8",53    log: "text/plain; charset=utf-8",54    sha256: "text/plain; charset=utf-8",55    parquet: "application/vnd.apache.parquet",56    zip: "application/zip",57    lean: "text/plain; charset=utf-8",58    html: "text/html; charset=utf-8",59    ipynb: "application/x-ipynb+json",60  };61  return map[ext ?? ""] ?? "application/octet-stream";62}6364const SKIP = [65  /(^|\/)\.DS_Store$/,66  /\.(aux|toc|out|bbl|blg|fls|fdb_latexmk|synctex\.gz|xdv|nav|snm|lof|lot)$/,67  /(^|\/)\.pytest_cache\//,68  /\.pyc$/,69  /(^|\/)\.ipynb_checkpoints\//,70];7172export interface UploadRecord {73  path: string;74  sha256: string;75  bytes: number;76  stage: StageKey;77}7879export async function hashFile(p: string): Promise<{ sha256: string; bytes: number }> {80  const { createReadStream } = await import("node:fs");81  const h = createHash("sha256");82  let bytes = 0;83  for await (const chunk of createReadStream(p)) {84    h.update(chunk as Buffer);85    bytes += (chunk as Buffer).length;86  }87  return { sha256: h.digest("hex"), bytes };88}8990/** Uploads every new/changed file under the workspace (excluding platform dirs). Tracks state in .autoscholar/uploaded.json. */91export async function syncArtifacts(92  workspace: string,93  stage: StageKey,94  upload: (path: string, kind: ArtifactKind, contentType: string, buf: Buffer) => Promise<void>,95  emit: (kind: string, message: string, data?: Record<string, unknown>) => void,96  opts: { synthetic?: boolean } = {},97): Promise<UploadRecord[]> {98  const statePath = join(workspace, ".autoscholar/uploaded.json");99  let state: Record<string, string> = {};100  try {101    state = JSON.parse(await readFile(statePath, "utf8"));102  } catch {103    /* first upload */104  }105  const files = await listFiles(workspace);106  const uploaded: UploadRecord[] = [];107  let skipped = 0;108  for (const rel of files) {109    if (SKIP.some((r) => r.test(rel))) continue;110    if (111      rel.startsWith("home/") ||112      rel.startsWith(".autoscholar/") ||113      rel.startsWith(".claude/") ||114      rel === "CLAUDE.md"115    )116      continue;117    const abs = join(workspace, rel);118    const st = await stat(abs);119    if (st.size > MAX_ARTIFACT_BYTES) {120      skipped++;121      emit("artifact.skipped", `skipped ${rel} (${st.size} bytes > limit)`, { path: rel, bytes: st.size });122      continue;123    }124    const buf = await readFile(abs);125    const sha256 = createHash("sha256").update(buf).digest("hex");126    if (state[rel] === sha256) continue;127    await upload(rel, kindFor(rel), contentTypeFor(rel), buf);128    state[rel] = sha256;129    uploaded.push({ path: rel, sha256, bytes: buf.length, stage });130  }131  await mkdir(join(workspace, ".autoscholar"), { recursive: true });132  await writeFile(statePath, JSON.stringify(state, null, 1));133  if (uploaded.length || skipped)134    emit(135      "artifact.sync",136      `uploaded ${uploaded.length} artifact(s) after ${stage}${skipped ? `, ${skipped} skipped` : ""}`,137      { stage, uploaded: uploaded.length, skipped, synthetic: !!opts.synthetic },138    );139  return uploaded;140}141142/** Writes provenance/checksums.sha256 for every artifact file (excluding itself and platform dirs). */143/**144 * Writes provenance/checksums.sha256. When an upload-state file is given, digests come from what was actually uploaded145 * (consistent with the artifact store even if files change afterwards); otherwise files are hashed in place.146 */147export async function writeChecksums(workspace: string, uploadStatePath?: string): Promise<string> {148  if (uploadStatePath) {149    try {150      const state = JSON.parse(await readFile(uploadStatePath, "utf8")) as Record<string, string>;151      const lines = Object.entries(state)152        .filter(([p]) => p !== "provenance/checksums.sha256")153        .sort(([a], [b]) => (a < b ? -1 : 1))154        .map(([p, sha]) => `${sha}  ${p}`);155      await mkdir(join(workspace, "provenance"), { recursive: true });156      const out = `${lines.join("\n")}\n`;157      await writeFile(join(workspace, "provenance/checksums.sha256"), out);158      return out;159    } catch {160      /* fall back to hashing */161    }162  }163  const files = (await listFiles(workspace)).filter(164    (f) =>165      !SKIP.some((r) => r.test(f)) &&166      !f.startsWith("home/") &&167      !f.startsWith(".autoscholar/") &&168      !f.startsWith(".claude/") &&169      f !== "CLAUDE.md" &&170      f !== "provenance/checksums.sha256",171  );172  const lines: string[] = [];173  const oversize: string[] = [];174  for (const f of files.sort()) {175    try {176      const st = await stat(join(workspace, f));177      if (st.size > MAX_ARTIFACT_BYTES) {178        oversize.push(`# not archived (oversize ${st.size} bytes): ${f}`);179        continue;180      }181      const { sha256 } = await hashFile(join(workspace, f));182      lines.push(`${sha256}  ${f}`);183    } catch (e) {184      oversize.push(`# unreadable: ${f} (${(e as Error).message.slice(0, 80)})`);185    }186  }187  await mkdir(join(workspace, "provenance"), { recursive: true });188  const out = `${[...lines, ...oversize].join("\n")}\n`;189  await writeFile(join(workspace, "provenance/checksums.sha256"), out);190  return out;191}192