spb/autoscholar

1import { createHash } from "node:crypto";2import { mkdir, readFile, stat, writeFile } from "node:fs/promises";3import { join } from "node:path";4import type { ArtifactKind, StageKey } from "@autoscholar/schemas";5import { listFiles } from "./latex.js";67export const MAX_ARTIFACT_BYTES = 256 * 1024 * 1024;89export function kindFor(path: string): ArtifactKind {10  if (path === "input/CLAUDE.md") return "instruction";11  if (path === "input/run.json") return "manifest";12  if (path.startsWith("plan/")) return "plan";13  if (path.startsWith("sources/")) return "source";14  if (path.startsWith("data/")) return "data";15  if (path.startsWith("code/")) return "code";16  if (path.startsWith("results/figures/")) return "figure";17  if (path.startsWith("results/tables/")) return "table";18  if (path.startsWith("results/")) return "result";19  if (path === "manuscript/main.pdf") return "pdf";20  if (path.startsWith("manuscript/")) return "manuscript";21  if (path.startsWith("review/")) return "review";22  if (path.startsWith("provenance/checksums")) return "checksum";23  if (path.startsWith("provenance/")) return "provenance";24  if (path.startsWith("logs/")) return "log";25  return "other";26}2728export function contentTypeFor(path: string): string {29  const ext = path.split(".").pop()?.toLowerCase();30  const map: Record<string, string> = {31    pdf: "application/pdf",32    md: "text/markdown; charset=utf-8",33    txt: "text/plain; charset=utf-8",34    json: "application/json",35    jsonl: "application/x-ndjson",36    tex: "text/x-tex; charset=utf-8",37    bib: "text/x-bibtex; charset=utf-8",38    cls: "text/x-tex; charset=utf-8",39    sty: "text/x-tex; charset=utf-8",40    bst: "text/plain; charset=utf-8",41    csv: "text/csv; charset=utf-8",42    png: "image/png",43    jpg: "image/jpeg",44    jpeg: "image/jpeg",45    svg: "image/svg+xml",46    py: "text/x-python; charset=utf-8",47    r: "text/plain; charset=utf-8",48    jl: "text/plain; charset=utf-8",49    sh: "text/x-shellscript; charset=utf-8",50    yaml: "text/yaml",51    yml: "text/yaml",52    toml: "text/plain; charset=utf-8",53    log: "text/plain; charset=utf-8",54    sha256: "text/plain; charset=utf-8",55    parquet: "application/vnd.apache.parquet",56    zip: "application/zip",57    lean: "text/plain; charset=utf-8",58    html: "text/html; charset=utf-8",59    ipynb: "application/x-ipynb+json",60  };61  return map[ext ?? ""] ?? "application/octet-stream";62}6364const SKIP = [65  /(^|\/)\.DS_Store$/,66  /\.(aux|toc|out|bbl|blg|fls|fdb_latexmk|synctex\.gz|xdv|nav|snm|lof|lot)$/,67  /(^|\/)\.pytest_cache\//,68  /\.pyc$/,69  /(^|\/)\.ipynb_checkpoints\//,70];7172export interface UploadRecord {73  path: string;74  sha256: string;75  bytes: number;76  stage: StageKey;77}7879export async function hashFile(p: string): Promise<{ sha256: string; bytes: number }> {80  const buf = await readFile(p);81  return { sha256: createHash("sha256").update(buf).digest("hex"), bytes: buf.length };82}8384/** Uploads every new/changed file under the workspace (excluding platform dirs). Tracks state in .autoscholar/uploaded.json. */85export async function syncArtifacts(86  workspace: string,87  stage: StageKey,88  upload: (path: string, kind: ArtifactKind, contentType: string, buf: Buffer) => Promise<void>,89  emit: (kind: string, message: string, data?: Record<string, unknown>) => void,90  opts: { synthetic?: boolean } = {},91): Promise<UploadRecord[]> {92  const statePath = join(workspace, ".autoscholar/uploaded.json");93  let state: Record<string, string> = {};94  try {95    state = JSON.parse(await readFile(statePath, "utf8"));96  } catch {97    /* first upload */98  }99  const files = await listFiles(workspace);100  const uploaded: UploadRecord[] = [];101  let skipped = 0;102  for (const rel of files) {103    if (SKIP.some((r) => r.test(rel))) continue;104    if (105      rel.startsWith("home/") ||106      rel.startsWith(".autoscholar/") ||107      rel.startsWith(".claude/") ||108      rel === "CLAUDE.md"109    )110      continue;111    const abs = join(workspace, rel);112    const st = await stat(abs);113    if (st.size > MAX_ARTIFACT_BYTES) {114      skipped++;115      emit("artifact.skipped", `skipped ${rel} (${st.size} bytes > limit)`, { path: rel, bytes: st.size });116      continue;117    }118    const buf = await readFile(abs);119    const sha256 = createHash("sha256").update(buf).digest("hex");120    if (state[rel] === sha256) continue;121    await upload(rel, kindFor(rel), contentTypeFor(rel), buf);122    state[rel] = sha256;123    uploaded.push({ path: rel, sha256, bytes: buf.length, stage });124  }125  await mkdir(join(workspace, ".autoscholar"), { recursive: true });126  await writeFile(statePath, JSON.stringify(state, null, 1));127  if (uploaded.length || skipped)128    emit(129      "artifact.sync",130      `uploaded ${uploaded.length} artifact(s) after ${stage}${skipped ? `, ${skipped} skipped` : ""}`,131      { stage, uploaded: uploaded.length, skipped, synthetic: !!opts.synthetic },132    );133  return uploaded;134}135136/** Writes provenance/checksums.sha256 for every artifact file (excluding itself and platform dirs). */137export async function writeChecksums(workspace: string): Promise<string> {138  const files = (await listFiles(workspace)).filter(139    (f) =>140      !SKIP.some((r) => r.test(f)) &&141      !f.startsWith("home/") &&142      !f.startsWith(".autoscholar/") &&143      !f.startsWith(".claude/") &&144      f !== "CLAUDE.md" &&145      f !== "provenance/checksums.sha256",146  );147  const lines: string[] = [];148  for (const f of files.sort()) {149    const { sha256 } = await hashFile(join(workspace, f));150    lines.push(`${sha256}  ${f}`);151  }152  await mkdir(join(workspace, "provenance"), { recursive: true });153  const out = `${lines.join("\n")}\n`;154  await writeFile(join(workspace, "provenance/checksums.sha256"), out);155  return out;156}157