spb/autoscholar
Public
1import { createHash } from "node:crypto";2import { createReadStream, createWriteStream } from "node:fs";3import { access, mkdir, readdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";4import { dirname, join, normalize, resolve, sep } from "node:path";5import type { Readable } from "node:stream";6import { pipeline } from "node:stream/promises";78export interface StoredObject {9 key: string;10 sha256: string;11 bytes: number;12}1314/** Write-once, checksum-verified object storage. Filesystem backend by default; S3 can implement the same interface. */15export interface StorageBackend {16 put(key: string, data: Buffer | Readable, expectedSha256?: string): Promise<StoredObject>;17 get(key: string): Promise<Buffer>;18 stream(key: string): Promise<Readable>;19 head(key: string): Promise<StoredObject | null>;20 exists(key: string): Promise<boolean>;21 copy(fromKey: string, toKey: string): Promise<StoredObject>;22 list(prefix: string): Promise<string[]>;23 remove(key: string): Promise<void>;24 absolutePath(key: string): string;25}2627export function sha256Hex(data: Buffer | string): string {28 return createHash("sha256").update(data).digest("hex");29}3031export async function sha256File(path: string): Promise<string> {32 const h = createHash("sha256");33 for await (const chunk of createReadStream(path)) h.update(chunk as Buffer);34 return h.digest("hex");35}3637export function assertSafeKey(key: string): string {38 const n = normalize(key).replace(/\\/g, "/");39 if (!n || n.startsWith("/") || n.startsWith("..") || n.split("/").includes("..") || n.includes("\0")) {40 throw new Error(`unsafe storage key: ${key}`);41 }42 return n;43}4445export class FilesystemStorage implements StorageBackend {46 readonly root: string;47 constructor(root: string) {48 this.root = resolve(root);49 }5051 absolutePath(key: string): string {52 const safe = assertSafeKey(key);53 const abs = resolve(this.root, safe);54 if (!abs.startsWith(this.root + sep) && abs !== this.root) throw new Error(`key escapes root: ${key}`);55 return abs;56 }5758 private sidecar(abs: string): string {59 return `${abs}.sha256`;60 }6162 async put(key: string, data: Buffer | Readable, expectedSha256?: string): Promise<StoredObject> {63 const abs = this.absolutePath(key);64 if (await this.exists(key)) {65 const existing = await this.head(key);66 if (existing && expectedSha256 && existing.sha256 !== expectedSha256) {67 throw new Error(`object exists with different digest: ${key}`);68 }69 if (existing) return existing; // write-once: identical content is idempotent70 }71 await mkdir(dirname(abs), { recursive: true });72 const tmp = `${abs}.tmp-${process.pid}-${Date.now()}`;73 const h = createHash("sha256");74 let bytes = 0;75 if (Buffer.isBuffer(data)) {76 h.update(data);77 bytes = data.length;78 await writeFile(tmp, data, { flag: "wx" });79 } else {80 await pipeline(81 data,82 async function* (src) {83 for await (const chunk of src) {84 const b = chunk as Buffer;85 h.update(b);86 bytes += b.length;87 yield b;88 }89 },90 createWriteStream(tmp, { flags: "wx" }),91 );92 }93 const sha256 = h.digest("hex");94 if (expectedSha256 && expectedSha256 !== sha256) {95 await rm(tmp, { force: true });96 throw new Error(`digest mismatch for ${key}: expected ${expectedSha256}, got ${sha256}`);97 }98 await rename(tmp, abs);99 await writeFile(this.sidecar(abs), `${sha256} ${key}\n`);100 return { key: assertSafeKey(key), sha256, bytes };101 }102103 async get(key: string): Promise<Buffer> {104 const abs = this.absolutePath(key);105 const buf = await readFile(abs);106 const meta = await this.head(key);107 if (meta && sha256Hex(buf) !== meta.sha256) throw new Error(`integrity failure reading ${key}`);108 return buf;109 }110111 async stream(key: string): Promise<Readable> {112 return createReadStream(this.absolutePath(key));113 }114115 async head(key: string): Promise<StoredObject | null> {116 const abs = this.absolutePath(key);117 try {118 const s = await stat(abs);119 let sha256: string;120 try {121 const side = await readFile(this.sidecar(abs), "utf8");122 sha256 = side.slice(0, 64);123 } catch {124 sha256 = await sha256File(abs);125 }126 return { key: assertSafeKey(key), sha256, bytes: s.size };127 } catch {128 return null;129 }130 }131132 async exists(key: string): Promise<boolean> {133 try {134 await access(this.absolutePath(key));135 return true;136 } catch {137 return false;138 }139 }140141 async copy(fromKey: string, toKey: string): Promise<StoredObject> {142 const buf = await this.get(fromKey);143 return this.put(toKey, buf);144 }145146 async list(prefix: string): Promise<string[]> {147 const abs = this.absolutePath(prefix);148 const out: string[] = [];149 const walk = async (dir: string, rel: string) => {150 let entries: import("node:fs").Dirent[];151 try {152 entries = (await readdir(dir, { withFileTypes: true })) as import("node:fs").Dirent[];153 } catch {154 return;155 }156 for (const e of entries) {157 const name = String(e.name);158 if (name.endsWith(".sha256") || name.includes(".tmp-")) continue;159 const r = rel ? `${rel}/${name}` : name;160 if (e.isDirectory()) await walk(join(dir, name), r);161 else out.push(`${assertSafeKey(prefix)}/${r}`.replace(/^\.\//, ""));162 }163 };164 await walk(abs, "");165 return out.sort();166 }167168 async remove(key: string): Promise<void> {169 const abs = this.absolutePath(key);170 await rm(abs, { force: true });171 await rm(this.sidecar(abs), { force: true });172 }173}174175export function runArtifactKey(runId: string, attemptNo: number, path: string): string {176 return assertSafeKey(`runs/${runId}/a${attemptNo}/${path}`);177}178179export function publicArtifactKey(publicId: string, versionNo: number, path: string): string {180 return assertSafeKey(`public/${publicId}/v${versionNo}/${path}`);181}182183export function getStorage(root = process.env.ARTIFACT_ROOT ?? "./data/artifacts"): FilesystemStorage {184 return new FilesystemStorage(root);185}186