spb/note-spboucher-ai

1/**2 * ZIP of Markdown (plain, Obsidian, our own export) or a Notion "Markdown & CSV" export → page tree.3 * Rules: folders mirror parents (`Foo.md` + `Foo/` → children of Foo); Notion `Name <32hex>` suffixes stripped; `Name.csv` → database4 * (`Name_all.csv` preferred, the filtered copy skipped) with `Name/<row title>.md` as row bodies; other files become attachments5 * when referenced; `references.bib` → references; `workspace.json` is ignored (metadata only).6 */7import path from "node:path";8import JSZip from "jszip";9import { createAttachment, createPage, mimeOf, updatePage, type ImportCtx, type Page } from "./pages";10import { importMarkdown, mdToDoc, parseFrontMatter, parseNotionHeader } from "./markdown";11import { importCsv } from "./csv";12import { importBibtexRefs } from "./bibtex";1314const NOTION_ID = /\s+[0-9a-f]{32}(?=(\.[a-z0-9]+)?$)/i;15const stripId = (seg: string) => seg.replace(NOTION_ID, "");16const normSeg = (s: string) => stripId(s).normalize("NFC").trim();17const normPath = (p: string) => p.split("/").filter(Boolean).map(normSeg).join("/");18const noExt = (p: string) => p.replace(/\.[^./]+$/, "");19const isNotionExport = (names: string[]) => names.some((n) => NOTION_ID.test(path.posix.basename(n)));2021interface Entry { raw: string; norm: string; file: JSZip.JSZipObject }22export async function importZip(ctx: ImportCtx, buf: Buffer, opts: { parentId?: string | null; filename?: string }) {23  const zip = await JSZip.loadAsync(buf);24  const entries: Entry[] = [];25  zip.forEach((rel, f) => { if (f.dir) return; const parts = rel.split("/"); if (parts.some((p) => p === "__MACOSX" || p === ".DS_Store" || p.startsWith("._"))) return; entries.push({ raw: rel, norm: normPath(rel), file: f }); });26  entries.sort((a, b) => a.norm.localeCompare(b.norm));27  const notion = isNotionExport(entries.map((e) => e.raw));28  // Strip a single common top-level folder (Notion exports and `zip -r folder` both produce one)29  const tops = new Set(entries.map((e) => e.norm.split("/")[0]));30  const strip = tops.size === 1 && entries.every((e) => e.norm.includes("/")) ? `${[...tops][0]}/` : "";31  for (const e of entries) e.norm = e.norm.slice(strip.length);32  const md = entries.filter((e) => /\.(md|markdown)$/i.test(e.norm) && !/\/?workspace\.json$/.test(e.norm));33  const csvs = entries.filter((e) => /\.csv$/i.test(e.norm));34  const bibs = entries.filter((e) => /\.bib$/i.test(e.norm));35  const others = entries.filter((e) => !md.includes(e) && !csvs.includes(e) && !bibs.includes(e) && !/\/?workspace\.json$/.test(e.norm));36  // Notion: prefer `_all.csv`, skip the filtered twin37  const csvKeys = new Map<string, Entry>();38  for (const c of csvs) { const key = noExt(c.norm).replace(/_all$/, ""); const cur = csvKeys.get(key); if (!cur || /_all\.csv$/i.test(c.norm)) csvKeys.set(key, c); }39  const total = md.length + csvKeys.size; let done = 0;40  const tick = async (msg: string) => { done++; if (done % 5 === 0 || done === total) await ctx.progress?.({ progress: Math.min(0.95, 0.05 + (0.9 * done) / Math.max(1, total)), message: msg }); };4142  // ---- pass 1: create every page (title only) so that links can resolve in pass 243  const pageByKey = new Map<string, Page>(); // key = normalised path without extension44  const dbByKey = new Map<string, { page: Page; csv: Entry }>();45  const folderPage = async (dirKey: string): Promise<string | null> => {46    if (!dirKey) return opts.parentId ?? null;47    const hit = pageByKey.get(dirKey); if (hit) return hit.id;48    const parent = await folderPage(path.posix.dirname(dirKey) === "." ? "" : path.posix.dirname(dirKey));49    const p = await createPage(ctx, { title: path.posix.basename(dirKey), parentId: parent, section: "notes", icon: "📁" });50    pageByKey.set(dirKey, p); ctx.paths.set(dirKey, p.id); return p.id;51  };52  const titleOf = async (e: Entry) => { const text = await e.file.async("string"); const { data, body } = parseFrontMatter(text); const nh = parseNotionHeader(body); return { text, title: String(data.title ?? nh.title ?? path.posix.basename(noExt(e.norm))).trim() || "Untitled", data }; };53  const mdText = new Map<string, { text: string; data: Record<string, unknown> }>();54  // shallow files first so parents exist before children55  md.sort((a, b) => a.norm.split("/").length - b.norm.split("/").length || a.norm.localeCompare(b.norm));56  for (const e of md) {57    const key = noExt(e.norm); const dirKey = path.posix.dirname(key) === "." ? "" : path.posix.dirname(key);58    // row pages of a database folder are created by the CSV import (matched by title) — defer them59    if (csvKeys.has(dirKey)) continue;60    const { text, title, data } = await titleOf(e); mdText.set(key, { text, data });61    const parent = await folderPage(dirKey);62    const p = await createPage(ctx, { title, parentId: parent, section: typeof data.section === "string" ? data.section : "notes", icon: typeof data.icon === "string" ? data.icon.slice(0, 16) : null });63    pageByKey.set(key, p); ctx.paths.set(key, p.id);64  }65  // ---- attachments on demand (relative to the referencing file, raw or normalised names)66  const attCache = new Map<string, { id: string; filename: string; mime: string; size: number } | null>();67  const resolverFor = (fromKey: string, pageId: string | null) => async (ref: string) => {68    const base = path.posix.dirname(fromKey) === "." ? "" : path.posix.dirname(fromKey);69    const candidates = [path.posix.normalize(path.posix.join(base, ref)), path.posix.normalize(ref)].map((c) => c.replace(/^(\.\/)+/, ""));70    for (const c of candidates) {71      const nc = normPath(c);72      const hit = others.find((o) => o.norm === nc || o.norm === c || o.raw.slice(strip.length) === c || normPath(o.raw.slice(strip.length)) === nc);73      if (!hit) continue;74      if (!attCache.has(hit.norm)) { const b = await hit.file.async("nodebuffer"); const a = await createAttachment(ctx, b, path.posix.basename(hit.norm), mimeOf(hit.norm), pageId); attCache.set(hit.norm, { id: a.id, filename: a.filename, mime: a.mime, size: a.size }); }75      return attCache.get(hit.norm)!;76    }77    return null;78  };79  const resolvePath = (fromKey: string) => (ref: string) => { const base = path.posix.dirname(fromKey) === "." ? "" : path.posix.dirname(fromKey); for (const c of [path.posix.join(base, ref), ref]) { const k = noExt(normPath(path.posix.normalize(c).replace(/^(\.\/)+/, ""))); const id = ctx.paths.get(k); if (id) return id; } return null; };8081  // ---- databases (CSV + row bodies from `<Name>/<Row title>.md`)82  for (const [key, csv] of csvKeys) {83    const dirKey = path.posix.dirname(key) === "." ? "" : path.posix.dirname(key);84    const parent = await folderPage(dirKey);85    const rowFiles = md.filter((e) => path.posix.dirname(noExt(e.norm)) === key);86    const bodies = new Map<string, Entry>(); for (const rf of rowFiles) { const { title } = await titleOf(rf); bodies.set(title.toLowerCase(), rf); }87    const text = await csv.file.async("string");88    const name = path.posix.basename(key);89    const existing = pageByKey.get(key); // `Name.md` next to `Name.csv` (Notion inline db page) → use its text as description90    const res = await importCsv(ctx, text, { name, parentId: parent, rowContent: async (title) => { const rf = bodies.get(title.toLowerCase()); if (!rf) return undefined; bodies.delete(title.toLowerCase()); const t = await rf.file.async("string"); const { body } = parseFrontMatter(t); const nh = parseNotionHeader(body); return mdToDoc(ctx, nh.body, { resolveFile: resolverFor(noExt(rf.norm), null), resolvePath: resolvePath(noExt(rf.norm)) }); } });91    dbByKey.set(key, { page: res.page, csv }); ctx.paths.set(key, res.page.id);92    if (existing) { // merge: the standalone page becomes redundant → keep as child "About" page only if it has a body93      const t = mdText.get(key)!; const { body } = parseFrontMatter(t.text); const nh = parseNotionHeader(body);94      if (nh.body.trim()) await updatePage(ctx, existing.id, { parentId: res.page.id, title: `${name} (notes)`, content: await mdToDoc(ctx, nh.body, { resolveFile: resolverFor(key, existing.id), resolvePath: resolvePath(key) }) }); else { const { db, schema } = await import("@/db"); const { eq } = await import("drizzle-orm"); await db.delete(schema.pages).where(eq(schema.pages.id, existing.id)); ctx.stats.pages--; }95    }96    // row subpages that did not match a CSV row → ordinary children of the database page97    for (const rf of bodies.values()) { const { text: t, title, data } = await titleOf(rf); const k = noExt(rf.norm); mdText.set(k, { text: t, data }); const p = await createPage(ctx, { title, parentId: res.page.id }); pageByKey.set(k, p); ctx.paths.set(k, p.id); }98    await tick(`Database ${name}: ${res.rows} rows`);99  }100  // ---- pass 2: bodies with resolved links/attachments101  for (const [key, p] of pageByKey) {102    const src = mdText.get(key); if (!src) continue; // folder pages103    await importMarkdown(ctx, src.text, { existingId: p.id, filename: path.posix.basename(key), resolveFile: resolverFor(key, p.id), resolvePath: resolvePath(key) });104    await tick(`Page ${p.title}`);105  }106  for (const b of bibs) { const text = await b.file.async("string"); const r = await importBibtexRefs(ctx, text, null); if (r.skipped.length) ctx.stats.skipped.push(...r.skipped.map((s) => `${b.norm}: ${s.citeKey} ${s.reason}`)); }107  const unreferenced = others.filter((o) => !attCache.has(o.norm));108  if (unreferenced.length) ctx.stats.skipped.push(`${unreferenced.length} unreferenced file(s) ignored: ${unreferenced.slice(0, 5).map((o) => o.norm).join(", ")}${unreferenced.length > 5 ? "…" : ""}`);109  const roots = [...pageByKey.values()].filter((p) => p.parentId === (opts.parentId ?? null));110  const firstRoot = roots[0] ?? [...dbByKey.values()][0]?.page ?? null;111  return { notion, pageId: firstRoot?.id ?? null, pages: pageByKey.size, databases: dbByKey.size };112}113