Repository Public
1/** Pure helpers over ProseMirror JSON (no DOM): plain text, block list, link/mention extraction, Markdown. Shared by server + client. */2import { NEXT_BLOCK_TYPES, isNextBlock, nextNodeText, nextNodeMarkdown, nodePageRef, nodeRefId, nodeAttachmentIds } from "./next-nodes";3export type PMNode = { type: string; attrs?: Record<string, any>; content?: PMNode[]; text?: string; marks?: { type: string; attrs?: Record<string, any> }[] };45export function nodeText(n: PMNode): string {6 if (n.text) return n.text;7 const next = nextNodeText(n); if (next !== null) return next; // 3.0 nodes (leaf atoms: their text lives in attrs)8 if (n.type === "hardBreak") return "\n";9 if (n.type === "mention") return `@${n.attrs?.label ?? ""}`;10 if (n.type === "math" || n.type === "inlineMath") return n.attrs?.latex ?? "";11 if (n.type === "blockMath") return n.attrs?.latex ?? "";12 return (n.content ?? []).map(nodeText).join((n.content ?? []).some(isBlock) ? "\n" : "");13}14const BLOCKS = new Set(["paragraph", "heading", "blockquote", "codeBlock", "bulletList", "orderedList", "listItem", "taskList", "taskItem", "table", "tableRow", "callout", "details", "detailsSummary", "detailsContent", "aiBlock", "blockMath", "horizontalRule", "image", "fileBlock", "youtube", "footnote", ...NEXT_BLOCK_TYPES]);15const isBlock = (n: PMNode) => BLOCKS.has(n.type);16export function docText(doc: PMNode | null | undefined): string { return doc ? nodeText(doc).replace(/\n{3,}/g, "\n\n").trim() : ""; }17export const wordCount = (s: string) => (s.match(/[\p{L}\p{N}]+(?:['’-][\p{L}\p{N}]+)*/gu) ?? []).length;1819/** Top-level-ish blocks with their text and a stable id (attrs.id if present). Used for chunking + citations. */20export function docBlocks(doc: PMNode | null | undefined): { id: string | null; type: string; heading: string | null; text: string }[] {21 const out: { id: string | null; type: string; heading: string | null; text: string }[] = []; let heading: string | null = null;22 const walk = (n: PMNode) => {23 if (n.type === "heading") { heading = nodeText(n).trim(); out.push({ id: n.attrs?.id ?? null, type: "heading", heading, text: heading }); return; }24 if (["paragraph", "blockquote", "codeBlock", "bulletList", "orderedList", "taskList", "table", "callout", "details", "aiBlock", "blockMath"].includes(n.type) || isNextBlock(n.type)) { const t = nodeText(n).trim(); if (t) out.push({ id: n.attrs?.id ?? null, type: n.type, heading, text: t }); return; }25 (n.content ?? []).forEach(walk);26 };27 (doc?.content ?? []).forEach(walk); return out;28}29export function extractPageRefs(doc: PMNode | null | undefined): string[] {30 const ids = new Set<string>();31 const walk = (n: PMNode) => { if (n.type === "mention" && n.attrs?.id && n.attrs?.kind !== "user") ids.add(n.attrs.id); const ref = nodePageRef(n); if (ref) ids.add(ref.pageId); for (const m of n.marks ?? []) if (m.type === "link" && typeof m.attrs?.href === "string") { const mm = m.attrs.href.match(/^\/p\/([0-9a-f-]{36})/); if (mm) ids.add(mm[1]); } (n.content ?? []).forEach(walk); };32 if (doc) walk(doc); return [...ids];33}3435/** ProseMirror JSON → Markdown (lossless enough for export/AI context). */36export function toMarkdown(doc: PMNode | null | undefined): string {37 if (!doc) return "";38 const inline = (n: PMNode): string => {39 if (n.type === "text") { let t = n.text ?? ""; for (const m of n.marks ?? []) { if (m.type === "bold") t = `**${t}**`; else if (m.type === "italic") t = `*${t}*`; else if (m.type === "code") t = `\`${t}\``; else if (m.type === "strike") t = `~~${t}~~`; else if (m.type === "link") t = `[${t}](${m.attrs?.href})`; else if (m.type === "highlight") t = `==${t}==`; } return t; }40 if (n.type === "hardBreak") return " \n"; if (n.type === "mention") return `[[${n.attrs?.label ?? n.attrs?.id}]]`; if (n.type === "inlineMath" || n.type === "math") return `$${n.attrs?.latex ?? ""}$`; if (n.type === "footnoteRef") return `[^${n.attrs?.id}]`;41 if (n.type === "citation") return nextNodeMarkdown(n) ?? "";42 return (n.content ?? []).map(inline).join("");43 };44 const block = (n: PMNode, depth = 0): string => {45 const pad = " ".repeat(depth);46 switch (n.type) {47 case "paragraph": return inline(n) + "\n";48 case "heading": return `${"#".repeat(n.attrs?.level ?? 1)} ${inline(n)}\n`;49 case "blockquote": return (n.content ?? []).map((c) => block(c)).join("").split("\n").filter(Boolean).map((l) => `> ${l}`).join("\n") + "\n";50 case "codeBlock": return `\`\`\`${n.attrs?.language ?? ""}\n${nodeText(n)}\n\`\`\`\n`;51 case "bulletList": return (n.content ?? []).map((li) => `${pad}- ${(li.content ?? []).map((c, i) => (i === 0 ? block(c, depth + 1).trimStart() : block(c, depth + 1))).join("")}`).join("");52 case "orderedList": return (n.content ?? []).map((li, i) => `${pad}${i + 1}. ${(li.content ?? []).map((c, j) => (j === 0 ? block(c, depth + 1).trimStart() : block(c, depth + 1))).join("")}`).join("");53 case "taskList": return (n.content ?? []).map((li) => `${pad}- [${li.attrs?.checked ? "x" : " "}] ${(li.content ?? []).map((c, i) => (i === 0 ? block(c, depth + 1).trimStart() : block(c, depth + 1))).join("")}`).join("");54 case "table": { const rows = (n.content ?? []).map((r) => (r.content ?? []).map((c) => inline(c).replace(/\|/g, "\\|").replace(/\n/g, " "))); if (!rows.length) return ""; return `| ${rows[0].join(" | ")} |\n| ${rows[0].map(() => "---").join(" | ")} |\n${rows.slice(1).map((r) => `| ${r.join(" | ")} |`).join("\n")}\n`; }55 case "horizontalRule": return "---\n";56 case "image": return `\n`;57 case "fileBlock": return `[📎 ${n.attrs?.name}](${n.attrs?.src})\n`;58 case "youtube": return `${n.attrs?.src}\n`;59 case "blockMath": return `$$\n${n.attrs?.latex ?? ""}\n$$\n`;60 case "callout": return `> [!${(n.attrs?.kind ?? "note").toUpperCase()}] ${n.attrs?.icon ?? ""}\n` + (n.content ?? []).map((c) => block(c)).join("").split("\n").filter(Boolean).map((l) => `> ${l}`).join("\n") + "\n";61 case "details": { const s = n.content?.find((c) => c.type === "detailsSummary"); const b = n.content?.find((c) => c.type === "detailsContent"); return `<details><summary>${s ? inline(s) : ""}</summary>\n\n${b ? (b.content ?? []).map((c) => block(c)).join("\n") : ""}\n</details>\n`; }62 case "aiBlock": return `> [!AI] ${n.attrs?.model ?? ""}\n` + (n.content ?? []).map((c) => block(c)).join("").split("\n").filter(Boolean).map((l) => `> ${l}`).join("\n") + "\n";63 case "footnote": return `[^${n.attrs?.id}]: ${(n.content ?? []).map(inline).join(" ")}\n`;64 default: { const next = nextNodeMarkdown(n); if (next !== null) return next; return (n.content ?? []).map((c) => block(c, depth)).join(""); }65 }66 };67 return (doc.content ?? []).map((n) => block(n)).join("\n").replace(/\n{3,}/g, "\n\n").trim() + "\n";68}69export const emptyDoc = (): PMNode => ({ type: "doc", content: [{ type: "paragraph" }] });7071/** Same block types as the editor's BlockId extension (src/components/editor/extensions/nodes.ts). */72const ID_TYPES = new Set(["paragraph", "heading", "blockquote", "codeBlock", "bulletList", "orderedList", "taskList", "table", "callout", "details", "aiBlock", "blockMath", "image", "fileBlock", "listItem", "taskItem", ...NEXT_BLOCK_TYPES]); // 3.0: list items and the new block atoms73const rid = () => { const a = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-"; let o = ""; const b = new Uint8Array(10); globalThis.crypto.getRandomValues(b); for (const x of b) o += a[x & 63]; return o; };74/**75 * Give every block a stable id when it lacks one (content created by a capture, an import, an agent or the API never went through the76 * editor, which is what normally assigns them). Search results and citations link to these ids, so they must exist before indexing.77 * Returns the same object when nothing was missing.78 */79export function ensureBlockIds<T extends PMNode | null | undefined>(doc: T, prev?: PMNode | null): { doc: T; changed: boolean } {80 if (!doc || typeof doc !== "object") return { doc, changed: false };81 // 3.0 — ids are stable across saves: a block that arrives without an id (a client that does not know the ids the server assigned,82 // or an editor that drops the id of a list item) gets back the id the same block had in the stored document (`prev`), matched by83 // type + text, in document order. Only a block that is really new gets a fresh id. Citations and task anchors therefore survive.84 const pool = new Map<string, string[]>(); const present = new Set<string>();85 const sig = (n: PMNode) => `${n.type}\u0000${nodeText(n).slice(0, 2000)}`;86 if (prev && typeof prev === "object") {87 const collect = (n: PMNode) => { if (ID_TYPES.has(n.type) && typeof n.attrs?.id === "string" && n.attrs.id) { const k = sig(n); const q = pool.get(k); if (q) q.push(n.attrs.id); else pool.set(k, [n.attrs.id]); } (n.content ?? []).forEach(collect); };88 collect(prev);89 const mark = (n: PMNode) => { if (typeof n.attrs?.id === "string") present.add(n.attrs.id); (n.content ?? []).forEach(mark); }; mark(doc as PMNode);90 }91 const seen = new Set<string>(); let changed = false;92 const fresh = (n: PMNode) => { const q = pool.get(sig(n)); while (q?.length) { const id = q.shift()!; if (!present.has(id) && !seen.has(id)) return id; } return rid(); };93 const walk = (n: PMNode): PMNode => {94 let out = n;95 if (ID_TYPES.has(n.type)) { const id = n.attrs?.id; if (typeof id !== "string" || !id || seen.has(id)) { const nid = fresh(n); seen.add(nid); out = { ...n, attrs: { ...(n.attrs ?? {}), id: nid } }; changed = true; } else seen.add(id); }96 if (n.content?.length) { const kids = n.content.map(walk); if (kids.some((k, i) => k !== n.content![i])) out = { ...out, content: kids }; }97 return out;98 };99 const next = walk(doc as PMNode);100 return { doc: (changed ? next : doc) as T, changed };101}102103// ---------- 3.0: structured extraction used by the page service (links, citations, tasks, journal)104/** Nearest enclosing block id while walking (so an inline citation or a mention is anchored to its paragraph / list item). */105function walkWithBlock(doc: PMNode | null | undefined, f: (n: PMNode, blockId: string | null) => void) {106 const walk = (n: PMNode, blockId: string | null) => { const own = ID_TYPES.has(n.type) && typeof n.attrs?.id === "string" && n.attrs.id ? (n.attrs.id as string) : blockId; f(n, own); (n.content ?? []).forEach((c) => walk(c, own)); };107 if (doc) walk(doc, null);108}109export interface DocRefs {110 /** page → page references: mentions, /p/<id> links, blockRef, canvasLink (first occurrence wins) */111 pages: { pageId: string; sourceBlockId: string | null; targetBlockId: string | null }[];112 /** citation nodes: reference id + the block that cites it */113 references: { refId: string; sourceBlockId: string | null }[];114 /** attachments referenced through the attrs of 3.0 nodes or /api/files/<id> sources */115 attachments: string[];116 /** every reference to a precise block of another page (blockRef with a blockId, /p/<id>#block-<blockId> links), not de-duplicated per page */117 blocks: { pageId: string; blockId: string; sourceBlockId: string | null }[];118}119export function extractRefs(doc: PMNode | null | undefined): DocRefs {120 const pages = new Map<string, DocRefs["pages"][number]>(); const references = new Map<string, DocRefs["references"][number]>(); const attachments = new Set<string>(); const blocks: DocRefs["blocks"] = [];121 const addPage = (pageId: string, sourceBlockId: string | null, targetBlockId: string | null) => { const k = pageId.toLowerCase(); if (!pages.has(k)) pages.set(k, { pageId: k, sourceBlockId, targetBlockId }); if (targetBlockId) blocks.push({ pageId: k, blockId: targetBlockId, sourceBlockId }); };122 const file = (v: unknown) => { const m = typeof v === "string" ? v.match(/^\/api\/files\/([0-9a-f-]{36})/i) : null; if (m) attachments.add(m[1].toLowerCase()); };123 walkWithBlock(doc, (n, blockId) => {124 if (n.type === "mention" && typeof n.attrs?.id === "string" && n.attrs.kind !== "user") addPage(n.attrs.id, blockId, null);125 for (const m of n.marks ?? []) if (m.type === "link" && typeof m.attrs?.href === "string") { const mm = m.attrs.href.match(/^\/p\/([0-9a-f-]{36})(?:#block-([A-Za-z0-9_-]{1,64}))?/); if (mm) addPage(mm[1], blockId, mm[2] ?? null); file(m.attrs.href); }126 const ref = nodePageRef(n); if (ref) addPage(ref.pageId, blockId, ref.blockId);127 const rid = nodeRefId(n); if (rid && !references.has(rid)) references.set(rid, { refId: rid, sourceBlockId: blockId });128 for (const a of nodeAttachmentIds(n)) attachments.add(a);129 file(n.attrs?.src);130 });131 return { pages: [...pages.values()], references: [...references.values()], attachments: [...attachments], blocks };132}133export interface DocTaskItem { blockId: string | null; taskId: string | null; checked: boolean; text: string; due: string | null; priority: "low" | "medium" | "high" | null }134const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;135/** Every `taskItem` of the document, in order, with its 3.0 attrs (`taskId`, `due`, `priority`). The text is the item's own first block. */136export function docTaskItems(doc: PMNode | null | undefined): DocTaskItem[] {137 const out: DocTaskItem[] = [];138 const walk = (n: PMNode) => {139 if (n.type === "taskItem") { const a = n.attrs ?? {}; const first = (n.content ?? []).find((c) => c.type !== "taskList" && c.type !== "bulletList" && c.type !== "orderedList"); out.push({ blockId: typeof a.id === "string" && a.id ? a.id : null, taskId: typeof a.taskId === "string" && UUID_RE.test(a.taskId) ? a.taskId.toLowerCase() : null, checked: Boolean(a.checked), text: (first ? nodeText(first) : "").replace(/\s+/g, " ").trim().slice(0, 500), due: typeof a.due === "string" && !isNaN(new Date(a.due).getTime()) ? a.due : null, priority: a.priority === "low" || a.priority === "medium" || a.priority === "high" ? a.priority : null }); }140 (n.content ?? []).forEach(walk);141 };142 if (doc) walk(doc); return out;143}144/** Blocks whose appearance in a document is a journal event: AI results inserted by the user (`aiBlock`) and decisions (`callout` kind `decision`). */145export function docMarkers(doc: PMNode | null | undefined): { ai: { blockId: string; model: string | null; action: string | null }[]; decisions: { blockId: string; text: string }[] } {146 const ai: { blockId: string; model: string | null; action: string | null }[] = []; const decisions: { blockId: string; text: string }[] = [];147 const walk = (n: PMNode) => {148 const id = typeof n.attrs?.id === "string" ? n.attrs.id : null;149 if (id && n.type === "aiBlock") ai.push({ blockId: id, model: typeof n.attrs?.model === "string" ? n.attrs.model : null, action: typeof n.attrs?.action === "string" ? n.attrs.action : null });150 if (id && n.type === "callout" && n.attrs?.kind === "decision") decisions.push({ blockId: id, text: nodeText(n).replace(/\s+/g, " ").trim().slice(0, 300) });151 (n.content ?? []).forEach(walk);152 };153 if (doc) walk(doc); return { ai, decisions };154}155