spb/note-spboucher-ai

1/**2 * Workspace / subtree → ZIP (streamed with jszip generateNodeStream; attachments are streamed from disk, never buffered).3 * Layout: <parents…>/<slug>.md, children under <slug>/, databases as <slug>.csv + <slug>/<row>.md, attachments/, references.bib, workspace.json.4 */5import fs from "node:fs";6import { PassThrough, Readable } from "node:stream";7import JSZip from "jszip";8import { absPath } from "@/lib/files/storage";9import { formatBibtex, fieldsOf } from "@/lib/research/bibtex";10import { slugify } from "@/lib/utils";11import { assignPaths, attachmentFile, loadExportSet, tagsOf, type ExportSet } from "./data";12import { pageMarkdown, zipResolvers } from "./markdown";13import { databaseCsv } from "./csv";14import { JSON_FORMAT } from "./json";1516/** jszip has no zip64 support → hard cap; the JSON export + per-subtree ZIPs are the escape hatch. */17export const ZIP_MAX_BYTES = Number(process.env.EXPORT_MAX_BYTES ?? 500 * 1024 * 1024);18export class ExportTooLarge extends Error { constructor(public bytes: number) { super(`Export too large (${(bytes / 1048576).toFixed(0)} MB > ${(ZIP_MAX_BYTES / 1048576).toFixed(0)} MB). Export subtrees separately or use the JSON backup.`); } }1920export function buildZip(set: ExportSet): { zip: JSZip; manifest: Record<string, unknown> } {21  const zip = new JSZip();22  const placed = assignPaths(set);23  const attById = new Map(set.attachments.map((a) => [a.id, a]));24  const attFiles = new Map<string, string>(); let bytes = 0;25  const addAttachment = (id: string): string | null => {26    const a = attById.get(id); if (!a) return null;27    if (!attFiles.has(id)) { const f = attachmentFile(a); let abs: string; try { abs = absPath(a.storagePath); } catch { return null; } if (!fs.existsSync(abs)) return null; bytes += a.size; if (bytes > ZIP_MAX_BYTES) throw new ExportTooLarge(bytes); zip.file(f, fs.createReadStream(abs), { date: a.createdAt }); attFiles.set(id, f); }28    return attFiles.get(id)!;29  };30  const dbByPage = new Map(set.databases.map((d) => [d.pageId, d]));31  const manifestPages: Record<string, unknown>[] = [];32  for (const { page, file, dir } of placed.values()) {33    const md = pageMarkdown(set, page, { placed, resolvers: zipResolvers(set, placed, file, addAttachment) });34    bytes += Buffer.byteLength(md); if (bytes > ZIP_MAX_BYTES) throw new ExportTooLarge(bytes);35    zip.file(file, md, { date: page.updatedAt });36    const d = dbByPage.get(page.id);37    if (d) { const { csv } = databaseCsv(set, d.id); zip.file(file.replace(/\.md$/, ".csv"), csv, { date: d.updatedAt }); }38    manifestPages.push({ id: page.id, title: page.title, type: page.type, section: page.section, parentId: page.parentId, databaseId: page.databaseId, icon: page.icon, path: file, dir, tags: tagsOf(set, page), createdAt: page.createdAt, updatedAt: page.updatedAt });39  }40  // attachments not referenced from any page content but linked to a page in the set41  for (const a of set.attachments) addAttachment(a.id);42  if (set.references.length) zip.file("references.bib", set.references.slice().sort((a, b) => a.citeKey.localeCompare(b.citeKey)).map((r) => r.bibtex ?? formatBibtex({ citeKey: r.citeKey, entryType: r.entryType, fields: fieldsOf(r) })).join("\n\n") + "\n");43  const manifest = {44    format: JSON_FORMAT, kind: set.rootId ? "subtree" : "workspace", exportedAt: new Date().toISOString(), app: "Note SPBoucher AI",45    workspace: { id: set.workspace.id, name: set.workspace.name, slug: set.workspace.slug }, rootId: set.rootId,46    counts: { pages: set.pages.length, databases: set.databases.length, rows: set.rows.length, attachments: attFiles.size, references: set.references.length, tasks: set.tasks.length },47    pages: manifestPages,48    databases: set.databases.map((d) => ({ id: d.id, pageId: d.pageId, name: d.name, description: d.description, csv: placed.get(d.pageId)?.file.replace(/\.md$/, ".csv"), properties: set.properties.filter((p) => p.databaseId === d.id).map((p) => ({ id: p.id, name: p.name, type: p.type, config: p.config, position: p.position })), views: set.views.filter((v) => v.databaseId === d.id).map((v) => ({ id: v.id, name: v.name, type: v.type, config: v.config, isDefault: v.isDefault })), rows: set.rows.filter((r) => r.databaseId === d.id).map((r) => ({ id: r.id, pageId: r.pageId, path: placed.get(r.pageId)?.file, values: r.values, position: r.position })) })),49    attachments: [...attFiles.entries()].map(([id, path]) => { const a = attById.get(id)!; return { id, path, filename: a.filename, mime: a.mime, size: a.size, sha256: a.sha256, pageId: a.pageId, createdAt: a.createdAt }; }),50    tasks: set.tasks.map((t) => ({ id: t.id, title: t.title, status: t.status, priority: t.priority, dueAt: t.dueAt, completedAt: t.completedAt, project: t.project, pageId: t.pageId })),51    tags: set.tags.map((t) => ({ id: t.id, name: t.name, color: t.color })),52  };53  zip.file("workspace.json", JSON.stringify(manifest, null, 2));54  return { zip, manifest };55}5657export async function exportZip(wsId: string, rootId?: string | null): Promise<{ stream: ReadableStream; filename: string }> {58  const set = await loadExportSet(wsId, rootId);59  const { zip } = buildZip(set);60  const root = rootId ? set.pages.find((p) => p.id === rootId) : null;61  const stamp = new Date().toISOString().slice(0, 10);62  const filename = `${slugify(root ? root.title : set.workspace.name) || "export"}-${stamp}.zip`;63  const pt = new PassThrough();64  zip.generateNodeStream({ type: "nodebuffer", streamFiles: true, compression: "DEFLATE", compressionOptions: { level: 6 } }).on("error", (e) => pt.destroy(e)).pipe(pt);65  return { stream: Readable.toWeb(pt) as unknown as ReadableStream, filename };66}67