SPB Git

Repository Public

text.ts 14.8 KB 155 lines typescript

1/** Pure helpers over ProseMirror JSON (no DOM): plain text, block list, link/mention extraction, Markdown. Shared by server + client. */2import { NEXT_BLOCK_TYPES, isNextBlock, nextNodeText, nextNodeMarkdown, nodePageRef, nodeRefId, nodeAttachmentIds } from "./next-nodes";3export type PMNode = { type: string; attrs?: Record<string, any>; content?: PMNode[]; text?: string; marks?: { type: string; attrs?: Record<string, any> }[] };45export function nodeText(n: PMNode): string {6  if (n.text) return n.text;7  const next = nextNodeText(n); if (next !== null) return next; // 3.0 nodes (leaf atoms: their text lives in attrs)8  if (n.type === "hardBreak") return "\n";9  if (n.type === "mention") return `@${n.attrs?.label ?? ""}`;10  if (n.type === "math" || n.type === "inlineMath") return n.attrs?.latex ?? "";11  if (n.type === "blockMath") return n.attrs?.latex ?? "";12  return (n.content ?? []).map(nodeText).join((n.content ?? []).some(isBlock) ? "\n" : "");13}14const BLOCKS = new Set(["paragraph", "heading", "blockquote", "codeBlock", "bulletList", "orderedList", "listItem", "taskList", "taskItem", "table", "tableRow", "callout", "details", "detailsSummary", "detailsContent", "aiBlock", "blockMath", "horizontalRule", "image", "fileBlock", "youtube", "footnote", ...NEXT_BLOCK_TYPES]);15const isBlock = (n: PMNode) => BLOCKS.has(n.type);16export function docText(doc: PMNode | null | undefined): string { return doc ? nodeText(doc).replace(/\n{3,}/g, "\n\n").trim() : ""; }17export const wordCount = (s: string) => (s.match(/[\p{L}\p{N}]+(?:['’-][\p{L}\p{N}]+)*/gu) ?? []).length;1819/** Top-level-ish blocks with their text and a stable id (attrs.id if present). Used for chunking + citations. */20export function docBlocks(doc: PMNode | null | undefined): { id: string | null; type: string; heading: string | null; text: string }[] {21  const out: { id: string | null; type: string; heading: string | null; text: string }[] = []; let heading: string | null = null;22  const walk = (n: PMNode) => {23    if (n.type === "heading") { heading = nodeText(n).trim(); out.push({ id: n.attrs?.id ?? null, type: "heading", heading, text: heading }); return; }24    if (["paragraph", "blockquote", "codeBlock", "bulletList", "orderedList", "taskList", "table", "callout", "details", "aiBlock", "blockMath"].includes(n.type) || isNextBlock(n.type)) { const t = nodeText(n).trim(); if (t) out.push({ id: n.attrs?.id ?? null, type: n.type, heading, text: t }); return; }25    (n.content ?? []).forEach(walk);26  };27  (doc?.content ?? []).forEach(walk); return out;28}29export function extractPageRefs(doc: PMNode | null | undefined): string[] {30  const ids = new Set<string>();31  const walk = (n: PMNode) => { if (n.type === "mention" && n.attrs?.id && n.attrs?.kind !== "user") ids.add(n.attrs.id); const ref = nodePageRef(n); if (ref) ids.add(ref.pageId); for (const m of n.marks ?? []) if (m.type === "link" && typeof m.attrs?.href === "string") { const mm = m.attrs.href.match(/^\/p\/([0-9a-f-]{36})/); if (mm) ids.add(mm[1]); } (n.content ?? []).forEach(walk); };32  if (doc) walk(doc); return [...ids];33}3435/** ProseMirror JSON → Markdown (lossless enough for export/AI context). */36export function toMarkdown(doc: PMNode | null | undefined): string {37  if (!doc) return "";38  const inline = (n: PMNode): string => {39    if (n.type === "text") { let t = n.text ?? ""; for (const m of n.marks ?? []) { if (m.type === "bold") t = `**${t}**`; else if (m.type === "italic") t = `*${t}*`; else if (m.type === "code") t = `\`${t}\``; else if (m.type === "strike") t = `~~${t}~~`; else if (m.type === "link") t = `[${t}](${m.attrs?.href})`; else if (m.type === "highlight") t = `==${t}==`; } return t; }40    if (n.type === "hardBreak") return "  \n"; if (n.type === "mention") return `[[${n.attrs?.label ?? n.attrs?.id}]]`; if (n.type === "inlineMath" || n.type === "math") return `$${n.attrs?.latex ?? ""}$`; if (n.type === "footnoteRef") return `[^${n.attrs?.id}]`;41    if (n.type === "citation") return nextNodeMarkdown(n) ?? "";42    return (n.content ?? []).map(inline).join("");43  };44  const block = (n: PMNode, depth = 0): string => {45    const pad = "  ".repeat(depth);46    switch (n.type) {47      case "paragraph": return inline(n) + "\n";48      case "heading": return `${"#".repeat(n.attrs?.level ?? 1)} ${inline(n)}\n`;49      case "blockquote": return (n.content ?? []).map((c) => block(c)).join("").split("\n").filter(Boolean).map((l) => `> ${l}`).join("\n") + "\n";50      case "codeBlock": return `\`\`\`${n.attrs?.language ?? ""}\n${nodeText(n)}\n\`\`\`\n`;51      case "bulletList": return (n.content ?? []).map((li) => `${pad}- ${(li.content ?? []).map((c, i) => (i === 0 ? block(c, depth + 1).trimStart() : block(c, depth + 1))).join("")}`).join("");52      case "orderedList": return (n.content ?? []).map((li, i) => `${pad}${i + 1}. ${(li.content ?? []).map((c, j) => (j === 0 ? block(c, depth + 1).trimStart() : block(c, depth + 1))).join("")}`).join("");53      case "taskList": return (n.content ?? []).map((li) => `${pad}- [${li.attrs?.checked ? "x" : " "}] ${(li.content ?? []).map((c, i) => (i === 0 ? block(c, depth + 1).trimStart() : block(c, depth + 1))).join("")}`).join("");54      case "table": { const rows = (n.content ?? []).map((r) => (r.content ?? []).map((c) => inline(c).replace(/\|/g, "\\|").replace(/\n/g, " "))); if (!rows.length) return ""; return `| ${rows[0].join(" | ")} |\n| ${rows[0].map(() => "---").join(" | ")} |\n${rows.slice(1).map((r) => `| ${r.join(" | ")} |`).join("\n")}\n`; }55      case "horizontalRule": return "---\n";56      case "image": return `![${n.attrs?.alt ?? ""}](${n.attrs?.src})\n`;57      case "fileBlock": return `[📎 ${n.attrs?.name}](${n.attrs?.src})\n`;58      case "youtube": return `${n.attrs?.src}\n`;59      case "blockMath": return `$$\n${n.attrs?.latex ?? ""}\n$$\n`;60      case "callout": return `> [!${(n.attrs?.kind ?? "note").toUpperCase()}] ${n.attrs?.icon ?? ""}\n` + (n.content ?? []).map((c) => block(c)).join("").split("\n").filter(Boolean).map((l) => `> ${l}`).join("\n") + "\n";61      case "details": { const s = n.content?.find((c) => c.type === "detailsSummary"); const b = n.content?.find((c) => c.type === "detailsContent"); return `<details><summary>${s ? inline(s) : ""}</summary>\n\n${b ? (b.content ?? []).map((c) => block(c)).join("\n") : ""}\n</details>\n`; }62      case "aiBlock": return `> [!AI] ${n.attrs?.model ?? ""}\n` + (n.content ?? []).map((c) => block(c)).join("").split("\n").filter(Boolean).map((l) => `> ${l}`).join("\n") + "\n";63      case "footnote": return `[^${n.attrs?.id}]: ${(n.content ?? []).map(inline).join(" ")}\n`;64      default: { const next = nextNodeMarkdown(n); if (next !== null) return next; return (n.content ?? []).map((c) => block(c, depth)).join(""); }65    }66  };67  return (doc.content ?? []).map((n) => block(n)).join("\n").replace(/\n{3,}/g, "\n\n").trim() + "\n";68}69export const emptyDoc = (): PMNode => ({ type: "doc", content: [{ type: "paragraph" }] });7071/** Same block types as the editor's BlockId extension (src/components/editor/extensions/nodes.ts). */72const ID_TYPES = new Set(["paragraph", "heading", "blockquote", "codeBlock", "bulletList", "orderedList", "taskList", "table", "callout", "details", "aiBlock", "blockMath", "image", "fileBlock", "listItem", "taskItem", ...NEXT_BLOCK_TYPES]); // 3.0: list items and the new block atoms73const rid = () => { const a = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-"; let o = ""; const b = new Uint8Array(10); globalThis.crypto.getRandomValues(b); for (const x of b) o += a[x & 63]; return o; };74/**75 * Give every block a stable id when it lacks one (content created by a capture, an import, an agent or the API never went through the76 * editor, which is what normally assigns them). Search results and citations link to these ids, so they must exist before indexing.77 * Returns the same object when nothing was missing.78 */79export function ensureBlockIds<T extends PMNode | null | undefined>(doc: T, prev?: PMNode | null): { doc: T; changed: boolean } {80  if (!doc || typeof doc !== "object") return { doc, changed: false };81  // 3.0 — ids are stable across saves: a block that arrives without an id (a client that does not know the ids the server assigned,82  // or an editor that drops the id of a list item) gets back the id the same block had in the stored document (`prev`), matched by83  // type + text, in document order. Only a block that is really new gets a fresh id. Citations and task anchors therefore survive.84  const pool = new Map<string, string[]>(); const present = new Set<string>();85  const sig = (n: PMNode) => `${n.type}\u0000${nodeText(n).slice(0, 2000)}`;86  if (prev && typeof prev === "object") {87    const collect = (n: PMNode) => { if (ID_TYPES.has(n.type) && typeof n.attrs?.id === "string" && n.attrs.id) { const k = sig(n); const q = pool.get(k); if (q) q.push(n.attrs.id); else pool.set(k, [n.attrs.id]); } (n.content ?? []).forEach(collect); };88    collect(prev);89    const mark = (n: PMNode) => { if (typeof n.attrs?.id === "string") present.add(n.attrs.id); (n.content ?? []).forEach(mark); }; mark(doc as PMNode);90  }91  const seen = new Set<string>(); let changed = false;92  const fresh = (n: PMNode) => { const q = pool.get(sig(n)); while (q?.length) { const id = q.shift()!; if (!present.has(id) && !seen.has(id)) return id; } return rid(); };93  const walk = (n: PMNode): PMNode => {94    let out = n;95    if (ID_TYPES.has(n.type)) { const id = n.attrs?.id; if (typeof id !== "string" || !id || seen.has(id)) { const nid = fresh(n); seen.add(nid); out = { ...n, attrs: { ...(n.attrs ?? {}), id: nid } }; changed = true; } else seen.add(id); }96    if (n.content?.length) { const kids = n.content.map(walk); if (kids.some((k, i) => k !== n.content![i])) out = { ...out, content: kids }; }97    return out;98  };99  const next = walk(doc as PMNode);100  return { doc: (changed ? next : doc) as T, changed };101}102103// ---------- 3.0: structured extraction used by the page service (links, citations, tasks, journal)104/** Nearest enclosing block id while walking (so an inline citation or a mention is anchored to its paragraph / list item). */105function walkWithBlock(doc: PMNode | null | undefined, f: (n: PMNode, blockId: string | null) => void) {106  const walk = (n: PMNode, blockId: string | null) => { const own = ID_TYPES.has(n.type) && typeof n.attrs?.id === "string" && n.attrs.id ? (n.attrs.id as string) : blockId; f(n, own); (n.content ?? []).forEach((c) => walk(c, own)); };107  if (doc) walk(doc, null);108}109export interface DocRefs {110  /** page → page references: mentions, /p/<id> links, blockRef, canvasLink (first occurrence wins) */111  pages: { pageId: string; sourceBlockId: string | null; targetBlockId: string | null }[];112  /** citation nodes: reference id + the block that cites it */113  references: { refId: string; sourceBlockId: string | null }[];114  /** attachments referenced through the attrs of 3.0 nodes or /api/files/<id> sources */115  attachments: string[];116  /** every reference to a precise block of another page (blockRef with a blockId, /p/<id>#block-<blockId> links), not de-duplicated per page */117  blocks: { pageId: string; blockId: string; sourceBlockId: string | null }[];118}119export function extractRefs(doc: PMNode | null | undefined): DocRefs {120  const pages = new Map<string, DocRefs["pages"][number]>(); const references = new Map<string, DocRefs["references"][number]>(); const attachments = new Set<string>(); const blocks: DocRefs["blocks"] = [];121  const addPage = (pageId: string, sourceBlockId: string | null, targetBlockId: string | null) => { const k = pageId.toLowerCase(); if (!pages.has(k)) pages.set(k, { pageId: k, sourceBlockId, targetBlockId }); if (targetBlockId) blocks.push({ pageId: k, blockId: targetBlockId, sourceBlockId }); };122  const file = (v: unknown) => { const m = typeof v === "string" ? v.match(/^\/api\/files\/([0-9a-f-]{36})/i) : null; if (m) attachments.add(m[1].toLowerCase()); };123  walkWithBlock(doc, (n, blockId) => {124    if (n.type === "mention" && typeof n.attrs?.id === "string" && n.attrs.kind !== "user") addPage(n.attrs.id, blockId, null);125    for (const m of n.marks ?? []) if (m.type === "link" && typeof m.attrs?.href === "string") { const mm = m.attrs.href.match(/^\/p\/([0-9a-f-]{36})(?:#block-([A-Za-z0-9_-]{1,64}))?/); if (mm) addPage(mm[1], blockId, mm[2] ?? null); file(m.attrs.href); }126    const ref = nodePageRef(n); if (ref) addPage(ref.pageId, blockId, ref.blockId);127    const rid = nodeRefId(n); if (rid && !references.has(rid)) references.set(rid, { refId: rid, sourceBlockId: blockId });128    for (const a of nodeAttachmentIds(n)) attachments.add(a);129    file(n.attrs?.src);130  });131  return { pages: [...pages.values()], references: [...references.values()], attachments: [...attachments], blocks };132}133export interface DocTaskItem { blockId: string | null; taskId: string | null; checked: boolean; text: string; due: string | null; priority: "low" | "medium" | "high" | null }134const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;135/** Every `taskItem` of the document, in order, with its 3.0 attrs (`taskId`, `due`, `priority`). The text is the item's own first block. */136export function docTaskItems(doc: PMNode | null | undefined): DocTaskItem[] {137  const out: DocTaskItem[] = [];138  const walk = (n: PMNode) => {139    if (n.type === "taskItem") { const a = n.attrs ?? {}; const first = (n.content ?? []).find((c) => c.type !== "taskList" && c.type !== "bulletList" && c.type !== "orderedList"); out.push({ blockId: typeof a.id === "string" && a.id ? a.id : null, taskId: typeof a.taskId === "string" && UUID_RE.test(a.taskId) ? a.taskId.toLowerCase() : null, checked: Boolean(a.checked), text: (first ? nodeText(first) : "").replace(/\s+/g, " ").trim().slice(0, 500), due: typeof a.due === "string" && !isNaN(new Date(a.due).getTime()) ? a.due : null, priority: a.priority === "low" || a.priority === "medium" || a.priority === "high" ? a.priority : null }); }140    (n.content ?? []).forEach(walk);141  };142  if (doc) walk(doc); return out;143}144/** Blocks whose appearance in a document is a journal event: AI results inserted by the user (`aiBlock`) and decisions (`callout` kind `decision`). */145export function docMarkers(doc: PMNode | null | undefined): { ai: { blockId: string; model: string | null; action: string | null }[]; decisions: { blockId: string; text: string }[] } {146  const ai: { blockId: string; model: string | null; action: string | null }[] = []; const decisions: { blockId: string; text: string }[] = [];147  const walk = (n: PMNode) => {148    const id = typeof n.attrs?.id === "string" ? n.attrs.id : null;149    if (id && n.type === "aiBlock") ai.push({ blockId: id, model: typeof n.attrs?.model === "string" ? n.attrs.model : null, action: typeof n.attrs?.action === "string" ? n.attrs.action : null });150    if (id && n.type === "callout" && n.attrs?.kind === "decision") decisions.push({ blockId: id, text: nodeText(n).replace(/\s+/g, " ").trim().slice(0, 300) });151    (n.content ?? []).forEach(walk);152  };153  if (doc) walk(doc); return { ai, decisions };154}155
Files main
Menu

Theme