spb/polyllm

1/**2 * Helpers for fast, stable streaming Markdown.3 *4 * `splitStreamingMarkdown` cuts a growing Markdown string into completed top-level blocks (rendered5 * once and memoized) and a live tail (re-rendered on every token). Cuts happen only on blank lines6 * outside code fences, and never between blocks that Markdown would join across a blank line7 * (loose lists, block quotes, indented continuations, tables), so the split output renders8 * identically to a single parse.9 *10 * `normalizeMath` rewrites the `\( … \)` / `\[ … \]` delimiters most models emit into the11 * `$ … $` / `$$ … $$` forms `remark-math` understands, leaving code spans and fences untouched.12 */13export interface MarkdownSplit {14  /** Completed blocks — each is a self-contained Markdown fragment. */15  blocks: string[];16  /** Still-growing tail (may be empty). */17  tail: string;18}1920const FENCE_RE = /^(\s{0,3})(`{3,}|~{3,})/;21const CONTINUATION_RE = /^(\s{2,}|\t|>|[-*+]\s|\d{1,3}[.)]\s|\|)/;22const LIST_OR_QUOTE_RE = /^(\s*)([-*+]\s|\d{1,3}[.)]\s|>)/;2324/** Whether `text` ends inside an open ``` / ~~~ fence. */25export function endsInsideFence(text: string): boolean {26  let open: string | null = null;27  for (const line of text.split("\n")) {28    const m = FENCE_RE.exec(line);29    if (!m) continue;30    const marker = m[2];31    if (!open) open = marker;32    else if (marker[0] === open[0] && marker.length >= open.length) open = null;33  }34  return open !== null;35}3637export function splitStreamingMarkdown(text: string, opts: { minTailChars?: number } = {}): MarkdownSplit {38  if (!text) return { blocks: [], tail: "" };39  const minTail = opts.minTailChars ?? 0;40  const lines = text.split("\n");41  const blocks: string[] = [];42  let current: string[] = [];43  let fence: string | null = null;4445  const flushIfSafe = (nextLine: string | undefined) => {46    // Only cut when the block is closed AND the next block does not continue this one.47    if (!current.length) return;48    if (fence) return;49    if (nextLine === undefined) return;50    const prevLast = [...current].reverse().find((l) => l.trim() !== "") ?? "";51    if (CONTINUATION_RE.test(nextLine)) return; // indented / list / quote / table continuation52    if (LIST_OR_QUOTE_RE.test(prevLast) && LIST_OR_QUOTE_RE.test(nextLine)) return;53    blocks.push(current.join("\n").replace(/\n+$/, ""));54    current = [];55  };5657  for (let i = 0; i < lines.length; i++) {58    const line = lines[i];59    const m = FENCE_RE.exec(line);60    if (m) {61      const marker = m[2];62      if (!fence) fence = marker;63      else if (marker[0] === fence[0] && marker.length >= fence.length) fence = null;64    }65    current.push(line);66    const isLast = i === lines.length - 1;67    if (isLast) break;68    // A blank line closes a block candidate (outside fences).69    if (line.trim() === "" && !fence) {70      // Skip following blank lines; peek at the first non-blank line.71      let j = i + 1;72      while (j < lines.length && lines[j].trim() === "") j++;73      flushIfSafe(lines[j]);74    }75  }76  let tail = current.join("\n");77  // Keep a minimum amount of live text so the caret doesn't jump between blocks too eagerly.78  while (tail.length < minTail && blocks.length) tail = `${blocks.pop()}\n\n${tail}`;79  return { blocks, tail };80}8182/** Rewrite `\( x \)` → `$x$` and `\[ x \]` → `$$x$$` outside code (inline spans and fences). */83export function normalizeMath(text: string): string {84  if (!text || !/\\[([]/.test(text)) return text;85  const out: string[] = [];86  const segments = text.split(/(```[\s\S]*?```|~~~[\s\S]*?~~~|`[^`\n]*`)/g);87  for (const seg of segments) {88    if (!seg) continue;89    if (seg.startsWith("```") || seg.startsWith("~~~") || (seg.startsWith("`") && seg.endsWith("`") && seg.length > 1)) {90      out.push(seg);91      continue;92    }93    out.push(seg.replace(/\\\[\s*([\s\S]*?)\s*\\\]/g, (_m, inner: string) => `$$\n${inner}\n$$`).replace(/\\\((.+?)\\\)/g, (_m, inner: string) => `$${inner.trim()}$`));94  }95  return out.join("");96}9798/** File extension for the code-block Download action. */99export function extensionForLanguage(lang: string | undefined): string {100  const l = (lang ?? "").toLowerCase();101  const map: Record<string, string> = {102    javascript: "js", js: "js", jsx: "jsx", typescript: "ts", ts: "ts", tsx: "tsx", python: "py", py: "py", bash: "sh", sh: "sh", shell: "sh", zsh: "sh", json: "json", yaml: "yml", yml: "yml", markdown: "md", md: "md", html: "html", css: "css", scss: "scss", sql: "sql", go: "go", rust: "rs", rs: "rs", java: "java", kotlin: "kt", kt: "kt", swift: "swift", ruby: "rb", rb: "rb", php: "php", c: "c", cpp: "cpp", "c++": "cpp", h: "h", csharp: "cs", cs: "cs", toml: "toml", xml: "xml", dockerfile: "Dockerfile", diff: "diff", text: "txt", plaintext: "txt", txt: "txt", r: "r", scala: "scala", lua: "lua", perl: "pl", dart: "dart", latex: "tex", tex: "tex", graphql: "graphql", ini: "ini", makefile: "mk", nginx: "conf", powershell: "ps1", ps1: "ps1",103  };104  return map[l] ?? "txt";105}106