Repository Public Research preview
1// Manuscript model: turns LaTeX or Markdown source into lines, sections, labels,2// references, citations, exhibits and sentences, each tied to a file and a line.3// Text processing only. Nothing here runs TeX, pandoc or any external program.45export const CITE = "\u27E6cite\u27E7";6export const REF = "\u27E6ref\u27E7";78export function detectFormat(file, text) {9 if (/\.(tex|ltx|latex)$/i.test(file || "")) return "latex";10 if (/\.(md|markdown|mdown|qmd|rmd)$/i.test(file || "")) return "markdown";11 return /\\documentclass|\\(?:sub)*section\*?\{|\\begin\{[A-Za-z*]+\}|\\(?:[a-zA-Z]*cite[a-zA-Z]*|ref|eqref|label)\{/.test(text) ? "latex" : "markdown";12}1314export function countWords(t) {15 return (t.match(/[A-Za-z\u00C0-\u024F][A-Za-z\u00C0-\u024F'\u2019-]*|\d[\d.,]*/g) || []).length;16}1718// ---------------------------------------------------------------- LaTeX text1920function stripLatexComment(line) {21 const m = /(^|[^\\])%/.exec(line);22 if (!m) return [line, ""];23 const cut = m.index + m[1].length;24 return [line.slice(0, cut), line.slice(cut + 1)];25}2627const CITE_CMD = /\\([a-zA-Z]*cite[a-zA-Z]*)\*?((?:\[[^\]]*\])*)\{([^}]*)\}/g;28const NOT_CITE = /^(?:set)?citestyle$|^bibcite$|^defcitealias$|^citeindextrue$|^citeindexfalse$/;29const REF_CMD = /\\(eqref|autoref|[cC]ref|[vV]ref|pageref|nameref|ref|subref|[cC]refrange|cpageref|labelcref)\*?\{([^}]*)\}(?:\{([^}]*)\})?|\\hyperref\[([^\]]*)\]/g;3031let MACROS = { ref: null, cite: null };3233/** Finds one-argument macros that wrap \ref or \cite, such as \newcommand{\fig}[1]{Figure~\ref{#1}}. */34function findMacros(L) {35 const ref = [], cite = [];36 for (const l of L) {37 if (!l.text || !/\\(?:re|provide)?newcommand|\\def\\|\\DeclareRobustCommand/.test(l.text)) continue;38 const m = /\\(?:(?:re|provide)?newcommand|DeclareRobustCommand)\*?\s*\{?\\([a-zA-Z]+)\}?\s*\[1\]\s*\{(.*)\}\s*$/.exec(l.text) || /\\def\s*\\([a-zA-Z]+)\s*#1\s*\{(.*)\}\s*$/.exec(l.text);39 if (!m) continue;40 if (/\\(?:eq|auto|c|C|v|page|name)?ref\*?\{#1\}/.test(m[2])) ref.push(m[1]);41 else if (/\\[a-zA-Z]*cite[a-zA-Z]*\*?(?:\[[^\]]*\])*\{#1\}/.test(m[2])) cite.push(m[1]);42 }43 const re = (names) => (names.length ? new RegExp(`\\\\(${names.join("|")})\\{([^}]*)\\}`, "g") : null);44 return { ref: re(ref), cite: re(cite), refNames: ref, citeNames: cite };45}4647export function detexLatex(s) {48 if (MACROS.ref) s = s.replace(MACROS.ref, ` ${REF} `);49 if (MACROS.cite) s = s.replace(MACROS.cite, ` ${CITE} `);50 return s51 .replace(CITE_CMD, (m, cmd) => (NOT_CITE.test(cmd) ? " " : ` ${CITE} `))52 .replace(REF_CMD, ` ${REF} `)53 .replace(/\\label\{[^}]*\}/g, "")54 .replace(/\\href\{[^}]*\}/g, "")55 .replace(/\\(?:url|includegraphics|input|include|subfile|bibliography|bibliographystyle|vspace|hspace|addbibresource|graphicspath|pagestyle|thispagestyle|setcounter|addtocounter|rule|setlength|captionsetup)\*?(?:\[[^\]]*\])*\{[^}]*\}(?:\{[^}]*\})?/g, " ")56 .replace(/\\(?:begin|end)\{[^}]*\}(?:\[[^\]]*\])?/g, " ")57 .replace(/(\d)\\,(?=\d)/g, "$1,")58 .replace(/(\d)\{,\}(?=\d)/g, "$1,")59 .replace(/\\%/g, "%").replace(/\\&/g, "&").replace(/\\#/g, "#").replace(/\\_/g, "_")60 .replace(/\\\$/g, "\u0001").replace(/\$/g, "").replace(/\u0001/g, "$")61 .replace(/\\(?:leq|le)(?![a-zA-Z])/g, "<=").replace(/\\(?:geq|ge)(?![a-zA-Z])/g, ">=")62 .replace(/\\pm(?![a-zA-Z])/g, "\u00B1").replace(/\\times(?![a-zA-Z])/g, "x")63 .replace(/\\(chi|beta|alpha|rho|sigma|mu|eta|delta|gamma|lambda|theta|tau|omega|phi|psi|kappa|epsilon|varepsilon)(?![a-zA-Z])/g, "$1")64 .replace(/\\\\(?:\[[^\]]*\])?/g, " ")65 .replace(/\\[,;:! ]/g, " ")66 .replace(/~/g, " ")67 .replace(/\\[a-zA-Z@]+\*?/g, " ")68 .replace(/[{}]/g, "")69 .replace(/\s+/g, " ")70 .trim();71}7273/** Reads a brace group whose opening brace sits just before `pos` on line `i`. May span lines. */74export function readBraced(L, i, pos, end = L.length) {75 let depth = 1, out = "", j = i, p = pos;76 const stop = Math.min(end, i + 80);77 while (j < stop) {78 const s = L[j].text;79 for (; p < s.length; p++) {80 const c = s[p];81 if (c === "\\") { out += c + (s[p + 1] ?? ""); p++; continue; }82 if (c === "{") depth++;83 else if (c === "}") { depth--; if (depth === 0) return { text: out, endI: j, endPos: p + 1, closed: true }; }84 out += c;85 }86 out += " "; j++; p = 0;87 }88 return { text: out, endI: Math.max(i, j - 1), endPos: 0, closed: false };89}9091const VERBATIM = /\\begin\{(verbatim\*?|lstlisting|minted|Verbatim|comment|filecontents\*?)\}/;9293/** Flattens a LaTeX file and its \input / \include children into one ordered list of lines. */94export function flattenLatex(file, text, reader, state = { seen: new Set(), unresolved: [], depth: 0 }, out = []) {95 state.seen.add(file);96 const lines = text.split(/\r?\n/);97 let verb = null;98 for (let n = 0; n < lines.length; n++) {99 const raw = lines[n];100 if (verb) {101 if (raw.includes(`\\end{${verb}}`)) verb = null;102 out.push({ file, line: n + 1, raw, text: "", comment: "", masked: true });103 continue;104 }105 const [code, comment] = stripLatexComment(raw);106 const v = VERBATIM.exec(code);107 if (v && !code.includes(`\\end{${v[1]}}`)) {108 verb = v[1];109 out.push({ file, line: n + 1, raw, text: "", comment, masked: true });110 continue;111 }112 out.push({ file, line: n + 1, raw, text: v ? "" : code, comment, masked: !!v });113 if (v) continue;114 const inc = /\\(?:input|include|subfile)\{([^}]+)\}/.exec(code);115 if (inc) {116 const name = inc[1].trim();117 const child = reader && state.depth < 8 ? reader(name) : null;118 if (child && child.text != null && !state.seen.has(child.file)) {119 state.depth++;120 flattenLatex(child.file, child.text, reader, state, out);121 state.depth--;122 } else {123 state.unresolved.push({ name, file, line: n + 1, reason: child?.reason ?? (reader ? "not found or already included" : "inline text: included files are not read") });124 }125 }126 }127 return { L: out, unresolved: state.unresolved };128}129130const FLOAT_ENV = /^(figure|table|sidewaystable|sidewaysfigure|wrapfigure|wraptable|longtable|subfigure|subtable)$/;131const TAB_ENV = /^(tabular|tabularx|tabulary|longtable|tabu|longtabu)$/;132const MATH_ENV = /^(equation|align|alignat|gather|multline|eqnarray|flalign|displaymath|subequations)$/;133const NUMBERED_MATH = /^(equation|align|alignat|gather|multline|eqnarray|flalign)$/;134const THM_ENV = /^(theorem|lemma|proposition|corollary|definition|assumption|hypothesis|remark|example|algorithm|listing|claim|conjecture)$/;135const NON_PROSE = /^\s*\\(?:title|author|date|affil|affiliation|address|email|thanks|keywords|maketitle|bibliography|bibliographystyle|printbibliography|tableofcontents|listoffigures|listoftables|newcommand|renewcommand|providecommand|usepackage|input|include|subfile|jel|pacs|newpage|clearpage|pagebreak|documentclass|setlength|onehalfspacing|doublespacing|singlespacing|centering|noindent\s*$)/;136const SEC_LEVEL = { part: 0, chapter: 0, section: 1, subsection: 2, subsubsection: 3, paragraph: 4 };137const TOK = /\\(begin|end)\{([^}]+)\}|\\label\{([^}]*)\}|\\caption(?:of\{[^}]*\})?\*?(?:\[[^\]]*\])?\{|\\(part|chapter|section|subsection|subsubsection|paragraph)(\*?)(?:\[[^\]]*\])?\{|\\appendix(?![a-zA-Z])|\\bibitem(?:\[[^\]]*\])?\{([^}]*)\}|\\\[|\\\]|\\abstract\{|\\title(?:\[[^\]]*\])?\{/g;138139function kindFromPrefix(name) {140 const p = name.split(/[:._-]/)[0].toLowerCase();141 if (/^fig/.test(p)) return "figure";142 if (/^(tab|tbl)/.test(p)) return "table";143 if (/^(eq|eqn)$/.test(p)) return "equation";144 if (/^(sec|app|ch|chap|sub|ssec)$/.test(p)) return "section";145 if (/^(thm|lem|prop|cor|def|hyp|alg|lst|ass)$/.test(p)) return "statement";146 return "other";147}148149function parseLatex(doc) {150 const L = doc.L;151 MACROS = findMacros(L);152 doc.macros = { ref: MACROS.refNames, cite: MACROS.citeNames };153 const b = L.findIndex((l) => /\\begin\{document\}/.test(l.text));154 const e = L.findIndex((l) => /\\end\{document\}/.test(l.text));155 const from = b >= 0 ? b + 1 : 0, to = e >= 0 ? e : L.length;156 doc.bodyRange = [from, to];157 const stack = [];158 let curSec = -1, appendix = false, inDisplay = false, lastSecI = -99, absRange = [-1, -1];159 for (let i = 0; i < L.length; i++) {160 const t = /\\title(?:\[[^\]]*\])?\{/.exec(L[i].text);161 if (t && doc.title == null) doc.title = detexLatex(readBraced(L, i, t.index + t[0].length).text);162 }163 for (let i = from; i < to; i++) {164 const l = L[i];165 l.body = true;166 const s = l.text;167 const startBlock = inDisplay || stack.some((x) => x.block);168 const startAbs = stack.some((x) => x.abs);169 const startFloat = [...stack].reverse().find((x) => x.float)?.float ?? null;170 let opened = false, heading = false, endAbs = startAbs, headEnd = 0;171 TOK.lastIndex = 0;172 let m;173 while ((m = TOK.exec(s))) {174 const after = m.index + m[0].length;175 if (m[1] === "begin") {176 const env = m[2], base = env.replace(/\*$/, "");177 const entry = { env, base, i };178 if (FLOAT_ENV.test(base)) {179 const parent = [...stack].reverse().find((x) => x.float)?.float ?? null;180 if (/^sub/.test(base) && parent) entry.float = parent;181 else {182 entry.float = { kind: /fig/.test(base) ? "figure" : "table", env, start: i, end: i, captions: [], labels: [], tabulars: [], closed: false };183 doc.floats.push(entry.float);184 }185 entry.owner = !(/^sub/.test(base) && parent);186 entry.block = true; opened = true;187 }188 if (TAB_ENV.test(base)) {189 const fl = entry.float ?? [...stack].reverse().find((x) => x.float)?.float ?? null;190 entry.tab = { env: base, start: i, pos: after, end: null, endPos: 0, float: fl };191 doc.tabulars.push(entry.tab);192 if (fl) fl.tabulars.push(entry.tab);193 entry.block = true; opened = true;194 }195 if (MATH_ENV.test(base)) {196 entry.math = true; entry.block = true; opened = true;197 if (NUMBERED_MATH.test(base) && !env.endsWith("*")) doc.counts.equations++;198 }199 if (base === "abstract") { entry.abs = true; endAbs = true; }200 if (base === "thebibliography") { entry.block = true; entry.bib = true; opened = true; }201 if (THM_ENV.test(base)) entry.thm = base;202 stack.push(entry);203 } else if (m[1] === "end") {204 let k = stack.length - 1;205 while (k >= 0 && stack[k].env !== m[2]) k--;206 if (k >= 0) {207 const entry = stack[k];208 if (entry.float && entry.owner) { entry.float.end = i; entry.float.closed = true; }209 if (entry.tab) { entry.tab.end = i; entry.tab.endPos = m.index; }210 stack.length = k;211 }212 } else if (m[3] !== undefined) {213 const name = m[3].trim();214 const fl = [...stack].reverse().find((x) => x.float)?.float ?? null;215 let kind;216 if (fl) kind = fl.kind;217 else if (inDisplay || stack.some((x) => x.math)) kind = "equation";218 else if (stack.some((x) => x.thm)) kind = "statement";219 else if (i === lastSecI || (i - lastSecI <= 2 && /^\s*\\label/.test(s))) kind = "section";220 else kind = kindFromPrefix(name);221 doc.labels.push({ name, kind, i, float: fl });222 if (fl) fl.labels.push(name);223 } else if (m[0].startsWith("\\caption")) {224 const fl = [...stack].reverse().find((x) => x.float)?.float ?? null;225 const r = readBraced(L, i, after, to);226 if (fl) fl.captions.push({ i, text: detexLatex(r.text) });227 } else if (m[4]) {228 const r = readBraced(L, i, after, to);229 doc.sections.push({ level: SEC_LEVEL[m[4]], title: detexLatex(r.text), i, appendix, numbered: !m[5], minor: m[4] === "paragraph" });230 headEnd = r.endI === i ? r.endPos : s.length;231 curSec = doc.sections.length - 1;232 heading = true; lastSecI = i;233 } else if (m[0].startsWith("\\appendix")) {234 appendix = true;235 } else if (m[6] !== undefined) {236 doc.bibitems.push({ key: m[6].trim(), i });237 } else if (m[0] === "\\[") {238 if (s[m.index - 1] !== "\\") { inDisplay = true; opened = true; }239 } else if (m[0] === "\\]") {240 if (s[m.index - 1] !== "\\") inDisplay = false;241 } else if (m[0] === "\\abstract{") {242 const r = readBraced(L, i, after, to);243 absRange = [i, r.endI];244 }245 }246 const endBlock = inDisplay || stack.some((x) => x.block);247 l.sec = curSec;248 l.abs = startAbs || endAbs || (i >= absRange[0] && i <= absRange[1]);249 l.float = startFloat ?? [...stack].reverse().find((x) => x.float)?.float ?? null;250 l.t = detexLatex(heading ? s.slice(headEnd) : s);251 l.prose = !l.masked && !startBlock && !endBlock && !opened && !NON_PROSE.test(s) && (!heading || l.t !== "");252 l.listStart = /^\s*\\item(?![a-zA-Z])/.test(s);253 if (l.abs) l.t = l.t.replace(/^abstract\s*/i, "");254 doc.counts.footnotes += (s.match(/\\footnote(?:text)?\s*\{/g) || []).length;255 }256 // citations, references, bibliography sources257 for (let i = 0; i < L.length; i++) {258 const s = L[i].text;259 if (!s) continue;260 let m;261 const src = /\\bibliography\{([^}]*)\}|\\addbibresource(?:\[[^\]]*\])?\{([^}]*)\}/g;262 while ((m = src.exec(s))) for (const n of (m[1] ?? m[2]).split(",")) if (n.trim()) doc.bibSources.push(n.trim());263 if (!L[i].body) continue;264 CITE_CMD.lastIndex = 0;265 while ((m = CITE_CMD.exec(s))) {266 if (NOT_CITE.test(m[1])) continue;267 const keys = m[3].split(",").map((k) => k.trim());268 if (keys.every((k) => k === "")) { doc.cites.push({ key: "", i, cmd: m[1] }); continue; }269 for (const k of keys) {270 if (k === "*") doc.citeAll = true;271 else if (k) doc.cites.push({ key: k, i, cmd: m[1] });272 }273 }274 if (MACROS.ref) for (const x of s.matchAll(MACROS.ref)) for (const nm of x[2].split(",")) if (nm.trim()) doc.refs.push({ name: nm.trim(), i, cmd: x[1] });275 if (MACROS.cite) for (const x of s.matchAll(MACROS.cite)) for (const k of x[2].split(",")) if (k.trim()) doc.cites.push({ key: k.trim(), i, cmd: x[1] });276 REF_CMD.lastIndex = 0;277 while ((m = REF_CMD.exec(s))) {278 const names = m[4] !== undefined ? [m[4]] : [...m[2].split(","), ...(m[3] ? m[3].split(",") : [])];279 for (const n of names) if (n.trim()) doc.refs.push({ name: n.trim(), i, cmd: m[1] ?? "hyperref" });280 }281 }282 for (const tab of doc.tabulars) parseTabular(doc, tab);283 MACROS = { ref: null, cite: null };284 for (const f of doc.floats) doc.counts[f.kind === "figure" ? "figures" : "tables"]++;285}286287export function countColumns(spec) {288 let n = 0, i = 0;289 const group = () => {290 if (spec[i] !== "{") return null;291 let d = 0;292 const st = i;293 do { if (spec[i] === "\\") i++; else if (spec[i] === "{") d++; else if (spec[i] === "}") d--; i++; } while (i < spec.length && d > 0);294 return spec.slice(st + 1, i - 1);295 };296 while (i < spec.length) {297 const c = spec[i];298 if (c === "*") { i++; const k = parseInt(group() ?? "", 10); const inner = group() ?? ""; n += (Number.isFinite(k) ? k : 0) * countColumns(inner); }299 else if ("@!><".includes(c)) { i++; group(); }300 else if (c === "[") { while (i < spec.length && spec[i] !== "]") i++; i++; }301 else if (c === "{") group();302 else if (/[A-Za-z]/.test(c)) { n++; i++; if (spec[i] === "[") { while (i < spec.length && spec[i] !== "]") i++; i++; } while (spec[i] === "{") group(); }303 else i++;304 }305 return n;306}307308const RULES = /\\(?:hline|toprule|midrule|bottomrule|hdashline|morecmidrules|endfirsthead|endhead|endfoot|endlastfoot)(?![a-zA-Z])|\\addlinespace(?:\[[^\]]*\])?|\\cmidrule(?:\([^)]*\))?(?:\[[^\]]*\])?\{[^}]*\}|\\cline\{[^}]*\}|\\specialrule\{[^}]*\}\{[^}]*\}\{[^}]*\}|\\noalign\{[^}]*\}|\\rowcolor(?:\[[^\]]*\])?\{[^}]*\}|\\(?:caption|label)(?:\[[^\]]*\])?\{[^}]*\}/g;309310function parseTabular(doc, tab) {311 const L = doc.L;312 tab.rows = []; tab.cols = null; tab.spec = null; tab.nested = false;313 if (tab.end == null) return;314 if (doc.tabulars.some((o) => o !== tab && o.start >= tab.start && o.end != null && o.end <= tab.end && (o.start > tab.start || o.pos > tab.pos))) { tab.nested = true; }315 let i = tab.start, p = tab.pos;316 const skipWs = () => { while (p < L[i].text.length && /\s/.test(L[i].text[p])) p++; };317 skipWs();318 if (L[i].text[p] === "[") { const c = L[i].text.indexOf("]", p); if (c >= 0) p = c + 1; skipWs(); }319 if (/^(tabularx|tabulary|tabu|longtabu)$/.test(tab.env) || L[tab.start].text.slice(0, tab.pos).endsWith("{tabular*}")) {320 if (L[i].text[p] === "{") { const r = readBraced(L, i, p + 1); i = r.endI; p = r.endPos; skipWs(); }321 }322 if (L[i].text[p] !== "{") return;323 const r = readBraced(L, i, p + 1);324 if (!r.closed) return;325 tab.spec = r.text.trim();326 tab.cols = countColumns(tab.spec);327 if (tab.nested) return;328 let buf = "";329 const owner = [];330 for (let j = r.endI; j <= tab.end; j++) {331 const s = L[j].text;332 const seg = s.slice(j === r.endI ? r.endPos : 0, j === tab.end ? tab.endPos : s.length);333 buf += seg + "\n";334 for (let k = 0; k <= seg.length; k++) owner.push(j);335 }336 const clean = buf.replace(RULES, (x) => " ".repeat(x.length));337 // rows end at a \\ that is not inside braces (\makecell{a\\b} stays one cell)338 const chunks = [];339 let startAt = 0, depth0 = 0;340 for (let k = 0; k < clean.length; k++) {341 const c = clean[k];342 if (c === "{") depth0++;343 else if (c === "}") depth0 = Math.max(0, depth0 - 1);344 else if (c === "\\") {345 if (clean[k + 1] === "\\" && depth0 === 0) {346 let z = k + 2;347 if (clean[z] === "*") z++;348 const opt = /^\s*\[[^\]]*\]/.exec(clean.slice(z, z + 40));349 if (opt) z += opt[0].length;350 chunks.push([startAt, k]);351 startAt = z;352 k = z - 1;353 } else k++;354 }355 }356 chunks.push([startAt, clean.length]);357 for (const [a, z] of chunks) {358 const row = clean.slice(a, z);359 if (!row.trim()) continue;360 const first = a + row.search(/\S/);361 let depth = 0, cells = 1;362 for (let k = 0; k < row.length; k++) {363 const c = row[k];364 if (c === "\\") { k++; continue; }365 if (c === "{") depth++;366 else if (c === "}") depth = Math.max(0, depth - 1);367 else if (c === "&" && depth === 0) cells++;368 }369 for (const mc of row.matchAll(/\\multicolumn\s*\{(\d+)\}/g)) cells += parseInt(mc[1], 10) - 1;370 tab.rows.push({ i: owner[first], cells, text: row.replace(/\s+/g, " ").trim() });371 }372}373374// ------------------------------------------------------------- Markdown text375376const XREF_PREFIX = "(?:fig|tbl|tab|eq|sec|lst)";377378export function detexMarkdown(s, numeric = false) {379 let t = s380 .replace(/!\[([^\]]*)\]\([^)]*\)(?:\{[^}]*\})?/g, "$1")381 .replace(/\[([^\]]+)\]\([^)]*\)/g, "$1")382 .replace(/\[\^[^\]]+\]/g, "")383 .replace(/`[^`]*`/g, " ")384 .replace(/\[[^\]]*@[^\]]*\]/g, (m) => (new RegExp(`@(?!${XREF_PREFIX}:)[A-Za-z]`).test(m) ? ` ${CITE} ` : ` ${REF} `))385 .replace(new RegExp(`(^|[\\s(;])@${XREF_PREFIX}:[\\w.:-]*\\w`, "g"), `$1${REF}`)386 .replace(/(^|[\s(;])-?@[A-Za-z](?:[\w:.-]*\w)?/g, `$1${CITE}`);387 if (numeric) t = t.replace(/\[\d+(?:\s*[-\u2013,]\s*\d+)*\]/g, ` ${CITE} `);388 return t389 .replace(/\{#[^}]*\}/g, "")390 .replace(/<\/?[a-zA-Z][^<>]*>/g, " ")391 .replace(/\\([%$&#_*|])/g, "$1")392 .replace(/\$/g, "")393 .replace(/(\*\*|__)(.+?)\1/g, "$2")394 .replace(/(^|[\s(])[*_]([^*_\s][^*_]*)[*_](?=$|[\s).,;:])/g, "$1$2")395 .replace(/^\s*(?:[-*+]|\d+[.)])\s+/, "")396 .replace(/^\s*>+\s?/, "")397 .replace(/\s+/g, " ")398 .trim();399}400401function splitPipeRow(s) {402 const t = s.replace(/`[^`]*`/g, "x").replace(/\\\|/g, "x").trim().replace(/^\|/, "").replace(/\|$/, "");403 return t.split("|");404}405const PIPE_SEP = /^\s*\|?\s*:?-+:?\s*(?:\|\s*:?-+:?\s*)*\|?\s*$/;406const MD_REFS_TITLE = /^(references|bibliography|works cited|literature cited|reference list)$/i;407const MD_CAPTION = /^\s*(?:\*\*|__)?(Table|Figure|Fig\.?)\s+(\d+(?:\.\d+)*)\s*(?:\*\*|__)?\s*(?:[:.\u2014\u2013]|-\s)/i;408409function parseMarkdown(doc) {410 const L = doc.L;411 const n = L.length;412 let i = 0;413 for (const l of L) { l.text = l.raw; l.comment = ""; l.body = true; l.prose = false; l.abs = false; l.sec = -1; l.t = ""; l.float = null; }414 // front matter415 if (n && /^---\s*$/.test(L[0].raw)) {416 let j = 1;417 while (j < n && !/^(---|\.\.\.)\s*$/.test(L[j].raw)) j++;418 if (j < n) {419 let block = null;420 for (let k = 1; k < j; k++) {421 const raw = L[k].raw;422 const kv = /^([A-Za-z_-]+):\s*(.*)$/.exec(raw);423 if (kv) {424 block = null;425 const key = kv[1].toLowerCase();426 const val = kv[2].trim().replace(/^["']|["']$/g, "");427 if (key === "title") doc.title = detexMarkdown(val);428 else if (key === "bibliography") { for (const b of val.replace(/^\[|\]$/g, "").split(",")) if (b.trim()) doc.bibSources.push(b.trim().replace(/^["']|["']$/g, "")); if (!val) block = "bibliography"; }429 else if (key === "abstract") {430 if (!val || /^[|>][+-]?$/.test(val)) block = "abstract";431 else { L[k].abs = true; L[k].prose = true; L[k].t = detexMarkdown(val); }432 }433 } else if (block === "abstract" && /^\s+\S/.test(raw)) { L[k].abs = true; L[k].prose = true; L[k].t = detexMarkdown(raw); }434 else if (block === "bibliography" && /^\s*-\s+\S/.test(raw)) doc.bibSources.push(raw.replace(/^\s*-\s+/, "").trim().replace(/^["']|["']$/g, ""));435 L[k].fm = true;436 }437 L[0].fm = true; L[j].fm = true;438 i = j + 1;439 }440 }441 doc.bodyRange = [i, n];442 // pass 1: structure443 let fence = null, inHtml = false, inMath = false, curSec = -1, refsLevel = 0, absUntilHeading = false, absPara = false, afterRefs = false;444 const kinds = new Array(n).fill("text");445 for (let k = 0; k < i; k++) kinds[k] = "fm";446 for (let k = i; k < n; k++) {447 const l = L[k];448 let s = l.raw;449 const f = /^\s*(```+|~~~+)/.exec(s);450 if (fence) { if (f && f[1][0] === fence[0] && f[1].length >= fence.length) fence = null; kinds[k] = "code"; l.text = ""; l.masked = true; continue; }451 if (f) { fence = f[1]; kinds[k] = "code"; l.text = ""; l.masked = true; continue; }452 if (inHtml) {453 const e = s.indexOf("-->");454 if (e < 0) { l.comment = s; s = ""; } else { l.comment = s.slice(0, e); s = s.slice(e + 3); inHtml = false; }455 }456 s = s.replace(/<!--([\s\S]*?)-->/g, (_, c) => { l.comment += c; return ""; });457 const o = s.indexOf("<!--");458 if (o >= 0) { l.comment += s.slice(o + 4); s = s.slice(0, o); inHtml = true; }459 l.text = s;460 const trimmed = s.trim();461 if (inMath) { kinds[k] = "math"; if (trimmed.includes("$$")) { inMath = false; labelsMd(doc, s, k, "equation"); } continue; }462 if (trimmed.startsWith("$$")) {463 kinds[k] = "math"; doc.counts.equations++;464 if (trimmed.length > 2 && trimmed.indexOf("$$", 2) >= 0) labelsMd(doc, s, k, "equation"); else inMath = true;465 continue;466 }467 const h = /^(#{1,6})\s+(.*?)\s*#*\s*$/.exec(s);468 if (h) {469 const level = h[1].length;470 const title = detexMarkdown(h[2]);471 kinds[k] = "heading";472 absUntilHeading = false; absPara = false;473 if (refsLevel && level <= refsLevel) { refsLevel = 0; afterRefs = true; }474 if (MD_REFS_TITLE.test(title)) { refsLevel = level; l.sec = curSec; continue; }475 if (/^abstract$/i.test(title) && doc.sections.every((x) => x.level === 1) && doc.sections.length <= 1) { absUntilHeading = true; l.sec = curSec; continue; }476 doc.sections.push({ level, title, i: k, appendix: afterRefs || /^appendi(x|ces)\b/i.test(title), numbered: true });477 curSec = doc.sections.length - 1;478 labelsMd(doc, h[2], k, "section");479 l.sec = curSec;480 continue;481 }482 l.sec = curSec;483 if (refsLevel) {484 kinds[k] = "refs";485 const r = /^\s*\[(\d+)\]:?\s+\S/.exec(s) || /^\s*(\d+)[.)]\s+\S/.exec(s);486 if (r) doc.numericRefs.push({ n: parseInt(r[1], 10), i: k });487 continue;488 }489 if (!trimmed) { absPara = false; kinds[k] = "blank"; continue; }490 if (/^\[\^[^\]]+\]:/.test(trimmed)) doc.counts.footnotes++;491 const ab = /^\s*(?:\*\*|__)Abstract[:.]?(?:\*\*|__)[:.]?\s*(.*)$/i.exec(s) || /^\s*Abstract[:.]\s+(.*)$/.exec(s);492 if (ab && doc.sections.length <= 1) { absPara = true; l.abs = true; l.absText = ab[1]; continue; }493 if (absUntilHeading || absPara) l.abs = true;494 if (s.includes("|") && k + 1 < n && PIPE_SEP.test(L[k + 1].raw) && L[k + 1].raw.includes("-") && (L[k + 1].raw.includes("|"))) {495 const table = { start: k, end: k, cols: splitPipeRow(L[k + 1].raw).length, rows: [], caption: null, labels: [] };496 let q = k;497 while (q < n && L[q].raw.trim() && L[q].raw.includes("|")) {498 kinds[q] = "table"; L[q].text = L[q].raw; L[q].sec = curSec;499 if (q !== k + 1) table.rows.push({ i: q, cells: splitPipeRow(L[q].raw).length, text: L[q].raw.trim() });500 q++;501 }502 table.end = q - 1;503 doc.mdTables.push(table);504 doc.counts.tables++;505 k = q - 1;506 continue;507 }508 if (kinds[k] === "table") continue;509 const cap = MD_CAPTION.exec(s);510 if (cap) {511 kinds[k] = "caption";512 doc.exhibitDefs.push({ kind: /^t/i.test(cap[1]) ? "table" : "figure", n: cap[2], i: k });513 labelsMd(doc, s, k, null);514 continue;515 }516 if (/^\s*(?:Table)?:\s+\S/.test(s)) { kinds[k] = "caption"; labelsMd(doc, s, k, "table"); continue; }517 const imgs = [...s.matchAll(/!\[([^\]]*)\]\(([^)]*)\)(\{[^}]*\})?/g)];518 if (imgs.length) {519 for (const im of imgs) {520 const lab = /#([A-Za-z]+:[^\s}]+)/.exec(im[3] || "");521 doc.mdFigures.push({ i: k, alt: im[1].trim(), label: lab ? lab[1] : null });522 doc.counts.figures++;523 }524 labelsMd(doc, s, k, "figure");525 if (!s.replace(/!\[[^\]]*\]\([^)]*\)(\{[^}]*\})?/g, "").trim()) { kinds[k] = "image"; continue; }526 } else labelsMd(doc, s, k, null);527 kinds[k] = "prose";528 }529 // attach captions and labels to pipe tables: a ": caption" line belongs to the530 // table above it when there is one, a "Table N." line to the table below it.531 const adjacent = (q, dir) => {532 for (let r = q + dir, steps = 0; r >= 0 && r < n && steps < 3; r += dir, steps++) {533 if (kinds[r] === "table") return doc.mdTables.find((tb) => r >= tb.start && r <= tb.end) ?? null;534 if (kinds[r] !== "blank") return null;535 }536 return null;537 };538 for (let q = i; q < n; q++) {539 if (kinds[q] !== "caption") continue;540 const above = adjacent(q, -1), below = adjacent(q, 1);541 const numbered = MD_CAPTION.test(L[q].raw);542 if (numbered && !/^\s*(?:\*\*|__)?T/i.test(L[q].raw)) continue;543 const owner = numbered ? below ?? above : above ?? below;544 if (owner && owner.caption == null) { owner.caption = q; owner.labels = doc.labels.filter((x) => x.i === q).map((x) => x.name); }545 }546 doc.numeric = doc.numericRefs.length > 0;547 // pass 2: text, citations, references548 const single = doc.sections.filter((x) => x.level === 1);549 if (single.length === 1 && doc.sections.some((x) => x.level > 1) && doc.sections[0] === single[0]) {550 single[0].isTitle = true;551 if (doc.title == null) doc.title = single[0].title;552 }553 for (let k = i; k < n; k++) {554 const l = L[k];555 const kind = kinds[k];556 l.kind = kind;557 if (kind === "code" || kind === "math" || kind === "blank" || kind === "refs" || kind === "heading") { if (kind === "heading") l.t = ""; continue; }558 const src = l.absText !== undefined ? l.absText : l.text;559 l.t = detexMarkdown(src, doc.numeric);560 l.prose = kind === "prose" || l.absText !== undefined;561 l.listStart = /^\s*(?:[-*+]|\d+[.)])\s+/.test(l.text);562 const scan = l.text.replace(/`[^`]*`/g, " ");563 for (const m of scan.matchAll(new RegExp(`(^|[^\\w@.])-?@(${XREF_PREFIX}:[\\w.:-]*\\w)`, "g"))) doc.refs.push({ name: m[2], i: k, cmd: "@" });564 for (const m of scan.matchAll(/(^|[^\w@.])-?@([A-Za-z](?:[\w:.#$%&+?<>~/-]*\w)?)/g)) {565 if (new RegExp(`^${XREF_PREFIX}:`).test(m[2])) continue;566 doc.cites.push({ key: m[2], i: k, cmd: "@" });567 }568 if (kind === "prose" || l.absText !== undefined) {569 if (doc.numeric) for (const m of scan.matchAll(/\[(\d+(?:\s*[-\u2013,]\s*\d+)*)\]/g)) {570 for (const part of m[1].split(",")) {571 const r = part.split(/[-\u2013]/).map((x) => parseInt(x.trim(), 10));572 if (r.length === 2 && r[1] >= r[0] && r[1] - r[0] < 200) for (let q = r[0]; q <= r[1]; q++) doc.numCites.push({ n: q, i: k });573 else if (Number.isFinite(r[0])) doc.numCites.push({ n: r[0], i: k });574 }575 }576 for (const m of l.t.matchAll(/\b(Tables?|Figures?|Figs?\.?)\s+(\d+(?:\.\d+)*)(?:\s*(?:and|&|,|-|\u2013)\s*(\d+(?:\.\d+)*))?/g)) {577 const kind2 = /^t/i.test(m[1]) ? "table" : "figure";578 doc.exhibitMentions.push({ kind: kind2, n: m[2], i: k });579 if (m[3]) doc.exhibitMentions.push({ kind: kind2, n: m[3], i: k });580 }581 }582 }583 doc.sections = doc.sections.filter((x, idx) => { x.idx = idx; return true; });584}585586function labelsMd(doc, s, i, forced) {587 for (const m of s.matchAll(/\{[^}]*#([A-Za-z][\w.:-]*)[^}]*\}/g)) {588 const name = m[1];589 if (!name.includes(":") && forced !== "section") continue;590 const pk = kindFromPrefix(name);591 doc.labels.push({ name, kind: pk !== "other" ? pk : forced ?? "other", i, float: null });592 }593}594595// ------------------------------------------------------------- Bibliography596597export function parseBib(text, file = "<bib>") {598 const entries = [], problems = [];599 const t = text.replace(/^\uFEFF/, "");600 if (/^\s*\[/.test(t)) {601 try {602 const arr = JSON.parse(t);603 const lines = t.split(/\r?\n/);604 for (const it of arr) {605 const key = String(it.id ?? "");606 const ln = lines.findIndex((x) => x.includes(`"${key}"`)) + 1 || 1;607 const fields = {};608 if (it.title) fields.title = String(it.title);609 if (Array.isArray(it.author) && it.author.length) fields.author = it.author.map((a) => a.family ?? a.literal ?? "").join(" and ");610 if (Array.isArray(it.editor) && it.editor.length) fields.editor = "x";611 const y = it.issued?.["date-parts"]?.[0]?.[0] ?? it.issued?.raw ?? it.issued?.literal;612 if (y != null) fields.year = String(y);613 if (it.DOI) fields.doi = String(it.DOI);614 entries.push({ key, type: String(it.type ?? "csl"), file, line: ln, raw: (lines[ln - 1] ?? "").trim(), fields });615 }616 } catch (e) {617 problems.push({ file, line: 1, message: `CSL JSON could not be parsed: ${e.message}` });618 }619 return { entries, problems };620 }621 const lineAt = (() => {622 const starts = [0];623 for (let k = 0; k < t.length; k++) if (t[k] === "\n") starts.push(k + 1);624 return (pos) => { let lo = 0, hi = starts.length - 1; while (lo < hi) { const mid = (lo + hi + 1) >> 1; if (starts[mid] <= pos) lo = mid; else hi = mid - 1; } return lo + 1; };625 })();626 const rawLines = t.split(/\r?\n/);627 const head = /@([a-zA-Z]+)\s*([{(])/g;628 let m;629 while ((m = head.exec(t))) {630 const lineStart = t.lastIndexOf("\n", m.index) + 1;631 if (t.slice(lineStart, m.index).includes("%")) continue;632 const type = m[1].toLowerCase();633 const line = lineAt(m.index);634 let p = m.index + m[0].length;635 if (type === "comment" || type === "preamble" || type === "string") {636 let d = 1;637 while (p < t.length && d > 0) { if (t[p] === "{" || t[p] === "(") d++; else if (t[p] === "}" || t[p] === ")") d--; p++; }638 head.lastIndex = p;639 continue;640 }641 const comma = t.indexOf(",", p);642 const close = t.indexOf(m[2] === "{" ? "}" : ")", p);643 if (comma < 0 || (close >= 0 && close < comma)) {644 const key = t.slice(p, close < 0 ? p : close).trim();645 entries.push({ key, type, file, line, raw: rawLines[line - 1].trim(), fields: {} });646 head.lastIndex = close < 0 ? p : close + 1;647 continue;648 }649 const key = t.slice(p, comma).trim();650 p = comma + 1;651 const fields = {};652 let ok = true;653 for (;;) {654 while (p < t.length && /[\s,]/.test(t[p])) p++;655 if (p >= t.length) { ok = false; break; }656 if (t[p] === "}" || t[p] === ")") { p++; break; }657 const nm = /^([A-Za-z][\w:.+-]*)\s*=\s*/.exec(t.slice(p, p + 200));658 if (!nm) { ok = false; break; }659 p += nm[0].length;660 let value = "";661 for (;;) {662 if (t[p] === "{") {663 let d = 0;664 const st = p;665 do { if (t[p] === "\\") p++; else if (t[p] === "{") d++; else if (t[p] === "}") d--; p++; } while (p < t.length && d > 0);666 if (d > 0) { ok = false; break; }667 value += t.slice(st + 1, p - 1);668 } else if (t[p] === '"') {669 let d = 0;670 const st = ++p;671 while (p < t.length && (t[p] !== '"' || d > 0)) { if (t[p] === "\\") p++; else if (t[p] === "{") d++; else if (t[p] === "}") d--; p++; }672 if (p >= t.length) { ok = false; break; }673 value += t.slice(st, p);674 p++;675 } else {676 const bare = /^[^\s,#})]+/.exec(t.slice(p, p + 200));677 if (!bare) { ok = false; break; }678 value += bare[0];679 p += bare[0].length;680 }681 while (p < t.length && /\s/.test(t[p])) p++;682 if (t[p] === "#") { p++; while (p < t.length && /\s/.test(t[p])) p++; continue; }683 break;684 }685 if (!ok) break;686 fields[nm[1].toLowerCase()] = value.replace(/\s+/g, " ").trim();687 }688 entries.push({ key, type, file, line, raw: rawLines[line - 1].trim(), fields });689 if (!ok) {690 problems.push({ file, line, message: `Entry "${key}" could not be parsed completely (unbalanced braces or quotes, or a field without "=").` });691 const next = t.indexOf("\n@", m.index + 1);692 head.lastIndex = next < 0 ? t.length : next + 1;693 } else head.lastIndex = p;694 }695 return { entries, problems };696}697698// ------------------------------------------------------------- Sentences699700const ABBR = /(?:\b(?:e\.g|i\.e|et al|cf|vs|Fig|Figs|Eq|Eqs|Sec|Tab|Ref|Refs|No|Dr|Prof|Mr|Mrs|Ms|St|approx|resp|ca|viz|al|pp|vol|ed|eds|Inc|Ltd|Co|Corp|U\.S|U\.K)|(?:^|[\s(])[A-Za-z])$/;701702function buildSentences(doc) {703 const L = doc.L;704 const paras = [];705 let cur = null;706 const flush = () => { if (cur) paras.push(cur); cur = null; };707 for (let i = doc.bodyRange[0] > 0 && doc.format === "markdown" ? 0 : doc.bodyRange[0]; i < doc.bodyRange[1]; i++) {708 const l = L[i];709 if (!l.prose || !l.t) { flush(); continue; }710 if (cur && (cur.sec !== l.sec || cur.abs !== !!l.abs || l.listStart)) flush();711 if (!cur) cur = { text: "", starts: [], idx: [], sec: l.sec, abs: !!l.abs };712 if (cur.text) cur.text += " ";713 cur.starts.push(cur.text.length);714 cur.idx.push(i);715 cur.text += l.t;716 }717 flush();718 const sentences = [];719 const cut = /[.!?]+["'\u201D\u2019)\]]*\s+(?=["'\u201C\u2018(\[]?[A-Z0-9\u27E6])/g;720 for (const p of paras) {721 let start = 0, m;722 cut.lastIndex = 0;723 const push = (a, z) => {724 const raw = p.text.slice(a, z);725 const lead = raw.search(/\S/);726 if (lead < 0) return;727 sentences.push({ text: raw.slice(lead).trimEnd(), para: p, off: a + lead });728 };729 while ((m = cut.exec(p.text))) {730 const before = p.text.slice(start, m.index);731 if (m[0][0] === "." && m[0][1] !== "." && ABBR.test(before)) continue;732 const stopLen = m[0].search(/\s/);733 push(start, m.index + stopLen);734 start = m.index + m[0].length;735 }736 push(start, p.text.length);737 }738 doc.paragraphs = paras;739 doc.sentences = sentences;740}741742/** Index into doc.L of the line holding character `pos` of a sentence. */743export function lineAt(sent, pos = 0) {744 const { starts, idx } = sent.para;745 const abs = sent.off + pos;746 let k = 0;747 while (k + 1 < starts.length && starts[k + 1] <= abs) k++;748 return idx[k];749}750751// ------------------------------------------------------------------- Entry752753/**754 * @param {{file?: string, text: string, format?: string, reader?: (name: string) => ({file: string, text: string}|{reason: string}|null)}} input755 */756export function parseManuscript({ file = "<inline>", text, format, reader = null }) {757 const fmt = !format || format === "auto" ? detectFormat(file, text) : format;758 const doc = {759 file, format: fmt, title: null, L: [], sections: [], labels: [], refs: [], cites: [], citeAll: false,760 floats: [], tabulars: [], bibitems: [], bibSources: [], unresolvedIncludes: [],761 mdTables: [], mdFigures: [], exhibitDefs: [], exhibitMentions: [], numericRefs: [], numCites: [], numeric: false,762 counts: { figures: 0, tables: 0, equations: 0, footnotes: 0 },763 bib: null, bodyRange: [0, 0], sentences: [], paragraphs: [],764 };765 if (fmt === "latex") {766 const flat = flattenLatex(file, text, reader);767 doc.L = flat.L;768 doc.unresolvedIncludes = flat.unresolved;769 parseLatex(doc);770 } else {771 doc.L = text.replace(/^\uFEFF/, "").split(/\r?\n/).map((raw, k) => ({ file, line: k + 1, raw }));772 parseMarkdown(doc);773 }774 // section word counts775 for (const s of doc.sections) { s.words = 0; s.wordsTotal = 0; }776 let total = 0, absWords = 0, absFirst = -1, absLast = -1;777 for (let i = 0; i < doc.L.length; i++) {778 const l = doc.L[i];779 if (!l.prose || !l.t) continue;780 const w = countWords(l.t);781 if (l.abs) { absWords += w; if (absFirst < 0) absFirst = i; absLast = i; continue; }782 total += w;783 if (l.sec >= 0 && doc.sections[l.sec]) doc.sections[l.sec].words += w;784 }785 for (let k = 0; k < doc.sections.length; k++) {786 const s = doc.sections[k];787 s.wordsTotal = s.words;788 for (let q = k + 1; q < doc.sections.length && doc.sections[q].level > s.level; q++) s.wordsTotal += doc.sections[q].words;789 }790 doc.words = total;791 doc.abstract = absFirst >= 0 ? { first: absFirst, last: absLast, words: absWords } : null;792 buildSentences(doc);793 return doc;794}795