Repository Public Research preview
1"""LaTeX source parser for arXiv e-prints (``raw_format == "latex"``).23Regex/scanner pipeline (pandoc ``-t plain`` was evaluated on real q-fin sources and rejected: it renders math4without ``$`` delimiters and drops citation keys, both of which must survive for ``protect()``):56archive → safe extraction → main file → ``\\input``/``\\include`` resolution (depth ≤ 5, comments stripped)7→ ``\\begin{document}``…``\\end{document}`` → cut at bibliography / ``\\appendix`` → abstract → math tokens8(inline ``$…$``, display ``$$…$$``) → floats → captions → sections → paragraphs → inline macro cleaning.910Metadata (title, authors, dates, licence) comes from the stub written by ``htr.ingest.arxiv_oai``.11"""12from __future__ import annotations1314import datetime as _dt15import gzip16import io17import re18import tarfile19import tempfile20import unicodedata21from dataclasses import dataclass, field22from pathlib import Path2324from htr.parse.jats import EXCLUDED_SECTION_TITLES, ParsedDocument, ParsedParagraph, section_role25from htr.parse.protect import protect26from htr.utils.text import normalize_text2728# Private-use delimiters (survive NFKC and contain no TeX-special characters)29M_OPEN, M_CLOSE = "\ue000", "\ue001" # math token M_OPEN<n>M_CLOSE30CAP = "\ue002" # caption marker CAP<n>CAP31ITEM = "\ue003" # list-item marker32V_OPEN, V_CLOSE = "\ue004", "\ue005" # verbatim token (urls)33TOKEN_RE = re.compile(f"{M_OPEN}(\\d+){M_CLOSE}")34VTOKEN_RE = re.compile(f"{V_OPEN}(\\d+){V_CLOSE}")3536MAX_INPUT_DEPTH = 537MAX_FILE_BYTES = 20 << 2038TEXT_SUFFIXES = {".tex", ".ltx", ".latex", ".txt", ".bbl", ""}3940DISPLAY_ENVS = ("equation", "align", "alignat", "eqnarray", "gather", "multline", "flalign", "displaymath", "dmath", "dgroup", "math")41FLOAT_ENVS = ("figure", "table", "wrapfigure", "wraptable", "sidewaysfigure", "sidewaystable", "algorithm", "figwindow", "SCfigure")42DROP_ENVS = ("tabular", "tabularx", "tabulary", "longtable", "verbatim", "Verbatim", "lstlisting", "minted", "tikzpicture", "picture", "pspicture",43 "thebibliography", "keywords", "keyword", "acknowledgments", "acknowledgements", "acknowledgment", "acknowledgement", "titlepage",44 "references", "algorithmic", "filecontents", "CCSXML", "IEEEkeywords", "highlights", "graphicalabstract")45LIST_ENVS = ("itemize", "enumerate", "description", "compactitem", "compactenum", "inparaenum", "list")46THEOREM_DEFAULTS = {47 "theorem": "Theorem", "thm": "Theorem", "lemma": "Lemma", "lem": "Lemma", "proposition": "Proposition", "prop": "Proposition",48 "corollary": "Corollary", "cor": "Corollary", "definition": "Definition", "defn": "Definition", "defi": "Definition", "remark": "Remark",49 "rem": "Remark", "assumption": "Assumption", "assum": "Assumption", "example": "Example", "conjecture": "Conjecture", "claim": "Claim",50 "hypothesis": "Hypothesis", "condition": "Condition", "axiom": "Axiom", "fact": "Fact", "observation": "Observation", "proof": "Proof",51 "note": "Note", "exercise": "Exercise", "problem": "Problem", "solution": "Solution", "notation": "Notation",52}53DROP_CMDS = ("footnote", "footnotetext", "footnotemark", "label", "thanks", "index", "vspace", "hspace", "includegraphics", "bibliographystyle",54 "pagestyle", "thispagestyle", "setcounter", "addtocounter", "setlength", "addtolength", "addcontentsline", "newcommand",55 "renewcommand", "providecommand", "DeclareMathOperator", "newtheorem", "newenvironment", "renewenvironment", "title", "author",56 "date", "address", "affiliation", "affil", "email", "keywords", "JEL", "jel", "JELclass", "urlstyle", "hypersetup", "numberwithin",57 "captionsetup", "marginpar", "todo", "nocite", "captionof", "institute", "titlerunning", "authorrunning", "subjclass", "pacs",58 "corref", "cortext", "fntext", "tnotetext", "ead", "markboth", "markright", "graphicspath", "usepackage", "documentclass",59 "phantom", "hphantom", "vphantom", "linenumbers", "bibliography", "newpage", "raisebox", "rule", "newline", "resizebox", "scalebox",60 "fbox", "doi", "orcid", "received", "accepted", "communicated", "dedication", "acknowledgments", "acknowledgements")61HARMLESS_BARE = {"noindent", "centering", "medskip", "bigskip", "smallskip", "newpage", "clearpage", "cleardoublepage", "maketitle", "em", "bf", "it",62 "rm", "sc", "sf", "tt", "sl", "small", "large", "Large", "LARGE", "huge", "Huge", "footnotesize", "scriptsize", "tiny", "normalsize",63 "hline", "indent", "par", "protect", "relax", "null", "hfill", "vfill", "linebreak", "pagebreak", "nolinebreak", "nopagebreak",64 "upshape", "itshape", "bfseries", "mdseries", "normalfont", "sffamily", "ttfamily", "rmfamily", "slshape", "scshape", "selectfont",65 "raggedright", "raggedleft", "tableofcontents", "listoffigures", "listoftables", "xspace", "sloppy", "fussy", "frenchspacing",66 "onehalfspacing", "doublespacing", "singlespacing", "baselineskip", "parindent", "parskip", "textwidth", "linewidth",67 "columnwidth", "hsize", "vsize", "displaystyle", "allowbreak", "unskip", "ignorespaces", "strut", "quad", "qquad", "enskip",68 "thinspace", "negthinspace", "smallbreak", "medbreak", "bigbreak", "endinput", "printbibliography", "frontmatter",69 "mainmatter", "backmatter", "twocolumn", "onecolumn", "IEEEpeerreviewmaketitle", "and", "vskip", "hskip", "kern", "today"}70SPECIAL_WORDS = {"LaTeX": "LaTeX", "TeX": "TeX", "ldots": "...", "dots": "...", "textellipsis": "...", "S": "§", "P": "¶", "ss": "ß", "o": "ø",71 "O": "Ø", "ae": "æ", "AE": "Æ", "oe": "œ", "OE": "Œ", "aa": "å", "AA": "Å", "l": "ł", "L": "Ł", "i": "ı", "textendash": "–",72 "textemdash": "—", "textquoteright": "’", "textquoteleft": "‘", "textregistered": "®", "copyright": "©", "pounds": "£",73 "euro": "€", "textdollar": "$", "textpercent": "%", "textbackslash": "\\", "textasciitilde": "~", "etal": "et al.",74 "eg": "e.g.", "ie": "i.e.", "etc": "etc.", "cf": "cf.", "vs": "vs."}75ACCENTS = {"'": "\u0301", "`": "\u0300", "^": "\u0302", '"': "\u0308", "~": "\u0303", "=": "\u0304", ".": "\u0307", "u": "\u0306",76 "v": "\u030c", "H": "\u030b", "c": "\u0327", "k": "\u0328", "r": "\u030a", "d": "\u0323", "b": "\u0331"}77CITE_RE = re.compile(r"\\(?:[Cc]ite[a-zA-Z]*|[Pp]arencite|[Tt]extcite|[Aa]utocite|[Ff]ootcite|[Ss]martcite|[Ss]upercite)\*?(?![a-zA-Z])")78REF_RE = re.compile(r"\\(?:ref|eqref|autoref|cref|Cref|pageref|vref|Vref|nameref|crefrange|subref)\*?(?![a-zA-Z])")79SECTION_RE = re.compile(r"\\(part|chapter|section|subsection|subsubsection)(\*?)(?![a-zA-Z])")808182@dataclass83class _State:84 math: list[tuple[str, bool]] = field(default_factory=list) # (content, is_display)85 verb: list[str] = field(default_factory=list)86 captions: list[str] = field(default_factory=list)87 macros: dict[str, str] = field(default_factory=dict)88 theorems: dict[str, str] = field(default_factory=lambda: dict(THEOREM_DEFAULTS))89 warnings: list[str] = field(default_factory=list)90 residue: int = 091 cites: list[str] = field(default_factory=list)9293 def warn(self, msg: str) -> None:94 if msg not in self.warnings:95 self.warnings.append(msg)969798# ------------------------------------------------------------------------------------ brace utilities99def read_group(s: str, i: int, open_c: str = "{", close_c: str = "}") -> tuple[str, int] | None:100 """If ``s[i]`` opens a group, return (content, index after the closing char); escapes are honoured."""101 if i >= len(s) or s[i] != open_c:102 return None103 depth, j = 0, i104 while j < len(s):105 c = s[j]106 if c == "\\":107 j += 2108 continue109 if c == open_c:110 depth += 1111 elif c == close_c:112 depth -= 1113 if depth == 0:114 return s[i + 1:j], j + 1115 j += 1116 return None117118119def _skip_ws(s: str, i: int, allow_newline: bool = True) -> int:120 while i < len(s) and s[i] in (" \t\n" if allow_newline else " \t"):121 if s[i] == "\n" and i + 1 < len(s) and s[i + 1] == "\n":122 break123 i += 1124 return i125126127def read_args(s: str, i: int, n_req: int | None = None) -> tuple[list[str], list[str], int]:128 """Read optional ``[..]`` and required ``{..}`` groups after a command ending at ``i``.129130 ``n_req=None``: greedily read adjacent groups (the first may follow whitespace). Returns (opts, reqs, end).131 """132 opts, reqs = [], []133 j = i134 first = True135 while True:136 k = _skip_ws(s, j) if first else j137 if k < len(s) and s[k] == "[" and (not reqs or n_req is None):138 g = read_group(s, k, "[", "]")139 if g is None:140 break141 opts.append(g[0])142 j = g[1]143 continue144 if k < len(s) and s[k] == "{" and (n_req is None or len(reqs) < n_req):145 g = read_group(s, k)146 if g is None:147 break148 reqs.append(g[0])149 j = g[1]150 first = False151 continue152 break153 return opts, reqs, j154155156def replace_commands(s: str, pattern: re.Pattern, fn, n_req: int | None = 1) -> str:157 """Replace each ``\\cmd[opts]{args}`` matched by ``pattern`` with ``fn(match, opts, reqs)``."""158 out, pos = [], 0159 for m in pattern.finditer(s):160 if m.start() < pos:161 continue162 opts, reqs, end = read_args(s, m.end(), n_req)163 out.append(s[pos:m.start()])164 out.append(fn(m, opts, reqs))165 pos = end166 out.append(s[pos:])167 return "".join(out)168169170def find_env(s: str, name: str, start: int = 0) -> tuple[int, int, int, int] | None:171 """Locate ``\\begin{name}``…``\\end{name}`` (same-name nesting aware): (begin, body_start, body_end, end)."""172 b = s.find(f"\\begin{{{name}}}", start)173 if b < 0:174 return None175 body = b + len(name) + 8176 depth, j = 1, body177 tag_b, tag_e = f"\\begin{{{name}}}", f"\\end{{{name}}}"178 while depth:179 nb, ne = s.find(tag_b, j), s.find(tag_e, j)180 if ne < 0:181 return b, body, len(s), len(s)182 if 0 <= nb < ne:183 depth += 1184 j = nb + len(tag_b)185 else:186 depth -= 1187 j = ne + len(tag_e)188 return b, body, j - len(tag_e), j189190191def replace_envs(s: str, names: tuple[str, ...], fn, starred: bool = True) -> str:192 for base in names:193 for name in ((base, base + "*") if starred else (base,)):194 pos = 0195 while True:196 loc = find_env(s, name, pos)197 if loc is None:198 break199 b, bs, be, e = loc200 rep = fn(name, s[bs:be])201 s = s[:b] + rep + s[e:]202 pos = b + len(rep)203 return s204205206# ------------------------------------------------------------------------------------ archive / files207def _safe_member(name: str) -> bool:208 p = Path(name)209 return bool(name) and not name.startswith(("/", "\\")) and ".." not in p.parts and not p.is_absolute()210211212def extract_archive(path: Path, dest: Path, st: _State) -> None:213 data = path.read_bytes()214 if data[:2] == b"\x1f\x8b":215 try:216 with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tf:217 _extract_tar(tf, dest, st)218 return219 except (tarfile.TarError, EOFError, OSError):220 data = gzip.decompress(data)221 elif len(data) > 262 and data[257:262] == b"ustar":222 with tarfile.open(fileobj=io.BytesIO(data), mode="r:") as tf:223 _extract_tar(tf, dest, st)224 return225 if data.lstrip()[:5] == b"%PDF-":226 raise ValueError("PDF-only submission: no LaTeX source")227 (dest / "main.tex").write_bytes(data)228229230def _extract_tar(tf: tarfile.TarFile, dest: Path, st: _State) -> None:231 root = dest.resolve()232 for m in tf.getmembers():233 if not m.isreg():234 continue235 if not _safe_member(m.name):236 st.warn(f"unsafe archive member rejected: {m.name[:80]}")237 continue238 target = (dest / m.name).resolve()239 if root not in target.parents:240 st.warn(f"unsafe archive member rejected: {m.name[:80]}")241 continue242 if Path(m.name).suffix.lower() not in TEXT_SUFFIXES or m.size > MAX_FILE_BYTES:243 continue244 f = tf.extractfile(m)245 if f is None:246 continue247 target.parent.mkdir(parents=True, exist_ok=True)248 target.write_bytes(f.read())249250251def read_text(p: Path, st: _State) -> str:252 raw = p.read_bytes()253 try:254 return raw.decode("utf-8")255 except UnicodeDecodeError:256 st.warn(f"non-UTF-8 source decoded as latin-1: {p.name}")257 return raw.decode("latin-1")258259260_MAIN_RE = re.compile(r"\\document(?:class|style)\b")261262263def find_main(files: list[Path], st: _State) -> Path:264 cands = []265 for f in files:266 if f.stat().st_size > MAX_FILE_BYTES:267 continue268 t = strip_comments(read_text(f, _State()))269 if _MAIN_RE.search(t) and "\\begin{document}" in t:270 cands.append((f, t))271 if not cands:272 raise ValueError("no main LaTeX file (\\documentclass + \\begin{document}) in archive")273 if len(cands) > 1:274 st.warn(f"{len(cands)} candidate main files; chose by \\input usage then size")275 stems = {f.stem for f in files}276277 def score(c: tuple[Path, str]) -> tuple[int, int]:278 n_inputs = sum(1 for m in INPUT_RE.finditer(c[1]) if Path(_input_name(m)).stem in stems)279 return n_inputs, c[0].stat().st_size280281 return max(cands, key=score)[0]282283284# --------------------------------------------------------------------------------- comments / inputs285_COMMENT_LINE = re.compile(r"^((?:[^\\%\n]|\\.)*)%.*$")286287288def strip_comments(text: str) -> str:289 """Remove ``%`` comments (not ``\\%``); a line that is only a comment disappears with its newline."""290 out = []291 for line in text.split("\n"):292 m = _COMMENT_LINE.match(line)293 if m is None:294 out.append(line)295 elif m.group(1).strip():296 out.append(m.group(1))297 text = "\n".join(out)298 text = re.sub(r"\\iffalse\b.*?\\fi\b", "", text, flags=re.DOTALL)299 return replace_envs(text, ("comment",), lambda n, b: "", starred=False)300301302INPUT_RE = re.compile(r"\\(?:(?:input|include|subfile)(?![a-zA-Z])|subimport\{([^}]*)\}|import\{([^}]*)\})\s*(?:\{([^}]+)\}|([^\s{}\\]+))")303304305def _input_name(m: re.Match) -> str:306 prefix = m.group(1) or m.group(2) or ""307 return (prefix + (m.group(3) or m.group(4) or "")).strip()308309310def resolve_inputs(text: str, base: Path, root: Path, st: _State, depth: int = 0, seen: set[str] | None = None) -> str:311 seen = seen if seen is not None else set()312313 def sub(m: re.Match) -> str:314 name = _input_name(m)315 if Path(name).suffix.lower() in (".sty", ".cls", ".def", ".tikz", ".pgf", ".pstex_t", ".pdf_t"):316 return ""317 if depth >= MAX_INPUT_DEPTH:318 st.warn(f"\\input depth > {MAX_INPUT_DEPTH}: {name} not expanded")319 return ""320 for cand in (base / name, base / f"{name}.tex", root / name, root / f"{name}.tex"):321 try:322 c = cand.resolve()323 except OSError:324 continue325 if c.is_file() and root.resolve() in c.parents and str(c) not in seen:326 seen.add(str(c))327 sub_text = strip_comments(read_text(c, st)).split("\\endinput")[0]328 return "\n" + resolve_inputs(sub_text, c.parent, root, st, depth + 1, seen) + "\n"329 st.warn(f"missing \\input file: {name}")330 return ""331332 return INPUT_RE.sub(sub, text)333334335# --------------------------------------------------------------------------------------- preamble336def collect_macros(preamble: str, st: _State) -> None:337 """Zero-argument user macros (``\\newcommand{\\x}{...}``, ``\\def\\x{...}``) and theorem names."""338 for m in re.finditer(r"\\(?:re)?newcommand\*?\s*\{?\\([a-zA-Z]+)\}?\s*(?=\{)", preamble):339 g = read_group(preamble, m.end())340 if g and "#" not in g[0] and len(g[0]) < 200:341 st.macros[m.group(1)] = g[0].replace("\\xspace", "")342 for m in re.finditer(r"\\def\s*\\([a-zA-Z]+)\s*(?=\{)", preamble):343 g = read_group(preamble, m.end())344 if g and "#" not in g[0] and len(g[0]) < 200:345 st.macros[m.group(1)] = g[0].replace("\\xspace", "")346 for m in re.finditer(r"\\newtheorem\*?\s*\{([^}]+)\}\s*(?:\[[^\]]*\])?\s*\{([^}]+)\}", preamble):347 st.theorems[m.group(1).strip()] = m.group(2).strip()348349350_STRUCTURAL = re.compile(r"\\begin|\\end|\$|\\\[|\\\]|\\\(|\\\)")351352353def expand_macros(text: str, st: _State, structural: bool) -> str:354 """Expand zero-argument user macros: ``structural=True`` → only those producing math/environment355 delimiters (e.g. ``\\be`` → ``\\begin{equation}``, run before math tokenisation); else the text macros."""356 names = [k for k, v in st.macros.items() if bool(_STRUCTURAL.search(v)) == structural]357 if not names:358 return text359 pat = re.compile(r"\\(" + "|".join(sorted(map(re.escape, names), key=len, reverse=True)) + r")(?![a-zA-Z])") # following space kept (xspace)360 for _ in range(2):361 text = pat.sub(lambda m: st.macros[m.group(1)], text)362 return text363364365# ------------------------------------------------------------------------------------------- math366def _math_token(st: _State, content: str, display: bool) -> str:367 content = re.sub(r"\\label\s*\{[^}]*\}", "", content)368 content = re.sub(r"(?<!\\)\$", "", content) # nested $…$ (e.g. \resizebox inside a display) would break $-delimiting369 content = re.sub(r"\s+", " ", content).strip()370 if not content:371 return " "372 st.math.append((content, display))373 tok = f"{M_OPEN}{len(st.math) - 1}{M_CLOSE}"374 return f" {tok} " if display else tok375376377def tokenize_math(s: str, st: _State) -> str:378 s = replace_envs(s, DISPLAY_ENVS, lambda n, b: _math_token(st, b, n not in ("math",)))379 s = re.sub(r"(?<!\\)\\\[(.*?)\\\]", lambda m: _math_token(st, m.group(1), True), s, flags=re.DOTALL)380 s = re.sub(r"(?<!\\)\\\((.*?)\\\)", lambda m: _math_token(st, m.group(1), False), s, flags=re.DOTALL)381 s = replace_commands(s, re.compile(r"\\ensuremath(?![a-zA-Z])"), lambda m, o, r: _math_token(st, r[0] if r else "", False))382 return _scan_dollars(s, st)383384385def _unescaped(s: str, i: int) -> bool:386 n, j = 0, i - 1387 while j >= 0 and s[j] == "\\":388 n += 1389 j -= 1390 return n % 2 == 0391392393def _scan_dollars(s: str, st: _State) -> str:394 out, i = [], 0395 while i < len(s):396 c = s[i]397 if c != "$" or not _unescaped(s, i):398 out.append(c)399 i += 1400 continue401 display = s.startswith("$$", i)402 delim = "$$" if display else "$"403 j = i + len(delim)404 while True:405 j = s.find(delim, j)406 if j < 0 or _unescaped(s, j):407 break408 j += 1409 blank = re.search(r"\n[ \t]*\n", s[i:j]) if j >= 0 else None410 if j < 0 or (blank and not display):411 st.warn("unbalanced $ delimiter: kept literally")412 out.append(c)413 i += 1414 continue415 out.append(_math_token(st, s[i + len(delim):j], display))416 i = j + len(delim)417 return "".join(out)418419420def restore_math(s: str, st: _State) -> str:421 def r(m: re.Match) -> str:422 content, display = st.math[int(m.group(1))]423 return f"$${content}$$" if display else f"${content}$"424 return TOKEN_RE.sub(r, s)425426427# ------------------------------------------------------------------------------- structure helpers428def cut_tail(body: str, st: _State) -> tuple[str, bool]:429 """Truncate at the bibliography or ``\\appendix`` (whichever comes first). Returns (body, appendix_dropped)."""430 marks = {"bibliography": [r"\\begin\{thebibliography\}", r"\\bibliography\s*\{", r"\\printbibliography", r"\\begin\{references\}"],431 "appendix": [r"\\appendix\b", r"\\begin\{appendix\}", r"\\begin\{appendices\}"]}432 hits = []433 for kind, pats in marks.items():434 for p in pats:435 m = re.search(p, body)436 if m:437 hits.append((m.start(), kind))438 appendix = any(k == "appendix" for _, k in hits)439 if appendix:440 st.warn("appendix dropped")441 if any(k == "bibliography" for _, k in hits):442 st.warn("bibliography dropped")443 return (body[:min(hits)[0]] if hits else body), appendix444445446def extract_abstract(text: str) -> tuple[str | None, str]:447 loc = find_env(text, "abstract")448 if loc:449 return text[loc[1]:loc[2]], text[:loc[0]] + "\n\n" + text[loc[3]:]450 m = re.search(r"\\abstract\s*(?=\{)", text)451 if m:452 g = read_group(text, m.end())453 if g:454 return g[0], text[:m.start()] + "\n\n" + text[g[1]:]455 return None, text456457458def extract_floats(s: str, st: _State) -> str:459 def fl(name: str, body: str) -> str:460 caps: list[str] = []461 replace_commands(body, re.compile(r"\\(?:sub)?caption\*?(?![a-zA-Z])"), lambda m, o, r: caps.append(r[0]) or "" if r else "")462 marks = []463 for c in caps:464 st.captions.append(c)465 marks.append(f"\n\n{CAP}{len(st.captions) - 1}{CAP}\n\n")466 return "".join(marks) or "\n\n"467 return replace_envs(s, FLOAT_ENVS, fl)468469470def structural_envs(s: str, st: _State) -> str:471 dropped: list[str] = []472 s = replace_envs(s, DROP_ENVS, lambda n, b: dropped.append(n) or "\n\n")473 if dropped:474 st.warn("dropped environments: " + ",".join(sorted(set(dropped))))475 for env, label in st.theorems.items():476 def th(n: str, body: str, label: str = label) -> str:477 g = read_group(body.lstrip(), 0, "[", "]")478 name = f" ({g[0]})" if g else ""479 rest = body.lstrip()[g[1]:] if g else body480 return f"\n\n{label}{name}. {rest}\n\n"481 s = replace_envs(s, (env,), th)482 s = re.sub(r"\\begin\{(?:" + "|".join(LIST_ENVS) + r")\}(?:\[[^\]]*\])?", "\n\n", s)483 s = re.sub(r"\\end\{(?:" + "|".join(LIST_ENVS) + r")\}", "\n\n", s)484 s = replace_commands(s, re.compile(r"\\item(?![a-zA-Z])"), lambda m, o, r: f"\n\n{ITEM}" + (o[0] + " " if o else ""), n_req=0)485 s = re.sub(r"\\begin\{(?:minipage|multicols|adjustbox|parbox)\}(?:\[[^\]]*\])*\{[^}]*\}", "\n\n", s)486 s = re.sub(r"\\(?:begin|end)\{[^}]+\}(?:\[[^\]]*\])?", "\n\n", s)487 s = re.sub(r"\\par(?![a-zA-Z])", "\n\n", s)488 return s489490491def brace_balance(s: str) -> int:492 s = re.sub(r"\\[{}]", "", s)493 return s.count("{") - s.count("}")494495496# ------------------------------------------------------------------------------- inline cleaning497def _accent(m: re.Match) -> str:498 mark = ACCENTS.get(m.group(1))499 base = m.group(2) or m.group(3) or ""500 if base == "\\i":501 base = "i"502 return unicodedata.normalize("NFC", base + mark) if mark and base else base503504505def clean_inline(s: str, st: _State) -> str:506 s = re.sub(r"\\\\(?:\*)?(?:\[[^\]]*\])?", " ", s) # line breaks first (so that \\{ is not read as \{)507 s = replace_commands(s, CITE_RE, lambda m, o, r: _cite(st, r[0] if r else ""))508 s = replace_commands(s, REF_RE, lambda m, o, r: "(ref)")509 s = replace_commands(s, re.compile(r"\\(?:url|nolinkurl|path)(?![a-zA-Z])"), lambda m, o, r: _verb(st, r[0] if r else ""))510 s = replace_commands(s, re.compile(r"\\href(?![a-zA-Z])"), lambda m, o, r: r[1] if len(r) > 1 else (_verb(st, r[0]) if r else ""), n_req=2)511 s = replace_commands(s, re.compile(r"\\(?:" + "|".join(DROP_CMDS) + r")\*?(?![a-zA-Z])"), lambda m, o, r: " ", n_req=None)512 s = re.sub(r"\\(paragraph|subparagraph)\*?\s*\{([^{}]*)\}", lambda m: m.group(2).strip() + ("" if m.group(2).strip()[-1:] in ".:?!" else ".") + " ", s)513 s = re.sub(r"\\([`'^\"~=.])\s*(?:\{(\\i|[A-Za-z]?)\}|(\\i|[A-Za-z]))", _accent, s)514 s = re.sub(r"\\([uvHckrdb])(?:\s*\{(\\i|[A-Za-z]?)\}|\s+(\\i|[A-Za-z])(?![A-Za-z]))", _accent, s)515 s = re.sub(r"\\(" + "|".join(sorted(SPECIAL_WORDS, key=len, reverse=True)) + r")(?![a-zA-Z])\s?(?:\{\})?", lambda m: SPECIAL_WORDS[m.group(1)], s)516 s = s.replace("\\%", "\ue010").replace("\\&", "&").replace("\\$", "$").replace("\\#", "#").replace("\\_", "_")517 s = s.replace("\\{", "\ue011").replace("\\}", "\ue012")518 s = re.sub(r"\\[ ,;:!/@]", " ", s).replace("\\-", "")519 for _ in range(4):520 s2 = re.sub(r"\\[a-zA-Z]+\*?(?:\[[^\]\[]*\])*\{([^{}]*)\}", r"\1", s)521 if s2 == s:522 break523 s = s2524 bare = re.findall(r"\\([a-zA-Z]+)\*?", s)525 st.residue += sum(1 for b in bare if b not in HARMLESS_BARE)526 s = re.sub(r"\\[a-zA-Z]+\*?", " ", s)527 s = s.replace("{", "").replace("}", "").replace("~", " ")528 s = s.replace("---", "—").replace("--", "–").replace("``", "“").replace("''", "”").replace("`", "‘")529 s = s.replace("\ue010", "%").replace("\ue011", "{").replace("\ue012", "}")530 s = VTOKEN_RE.sub(lambda m: st.verb[int(m.group(1))], s)531 s = normalize_text(s.replace("\n", " "))532 return s533534535def _cite(st: _State, keys: str) -> str:536 ks = [k.strip() for k in keys.split(",") if k.strip()]537 if not ks:538 return ""539 c = "[" + ",".join(ks) + "]"540 st.cites.append(c)541 return c542543544def _verb(st: _State, u: str) -> str:545 st.verb.append(u.strip())546 return f"{V_OPEN}{len(st.verb) - 1}{V_CLOSE}"547548549# ------------------------------------------------------------------------------------- sections550@dataclass551class _Block:552 sec_id: str553 title: str | None554 role: str555 depth: int556 text: str557 flags: list[str]558 cites: list[str] = field(default_factory=list)559560561def split_sections(body: str, has_chapters: bool, st: _State) -> list[tuple[str | None, int, str]]:562 """[(raw_title, depth, raw_segment)]; the leading segment has title None and depth 1."""563 out: list[tuple[str | None, int, str]] = []564 pos, cur_title, cur_depth = 0, None, 1565 base = {"part": 1, "chapter": 1, "section": 2 if has_chapters else 1, "subsection": 3 if has_chapters else 2, "subsubsection": 4 if has_chapters else 3}566 for m in SECTION_RE.finditer(body):567 if m.start() < pos:568 continue569 _, reqs, end = read_args(body, m.end(), 1)570 out.append((cur_title, cur_depth, body[pos:m.start()]))571 cur_title, cur_depth, pos = (reqs[0] if reqs else ""), base[m.group(1)], end572 out.append((cur_title, cur_depth, body[pos:]))573 return out574575576def _is_display_only(raw: str, st: _State) -> bool:577 toks = TOKEN_RE.findall(raw)578 rest = re.sub(r"[\s.,;:]", "", TOKEN_RE.sub("", raw))579 return bool(toks) and not rest and all(st.math[int(t)][1] for t in toks)580581582def blocks_from_sections(sections: list[tuple[str | None, int, str]], st: _State) -> list[_Block]:583 blocks: list[_Block] = []584 stack: list[tuple[int, str]] = [] # (depth, role)585 skip_depth: int | None = None586 for n, (raw_title, depth, seg) in enumerate(sections):587 title = restore_math(clean_inline(raw_title, st), st) if raw_title is not None else None588 if skip_depth is not None and depth > skip_depth:589 continue590 skip_depth = None591 if title and EXCLUDED_SECTION_TITLES.search(title):592 skip_depth = depth593 st.warn(f"section excluded: {title[:60]}")594 continue595 while stack and stack[-1][0] >= depth:596 stack.pop()597 role = section_role(title, depth) if title is not None else "other"598 if role == "other" and stack and stack[-1][1] != "other":599 role = stack[-1][1]600 if title is not None:601 stack.append((depth, role))602 sid = "body" if title is None else f"s{n}"603 for chunk in re.split(r"\n[ \t]*\n+", seg):604 _add_chunk(blocks, chunk, sid, title, role, depth, st)605 return blocks606607608def _add_chunk(blocks: list[_Block], chunk: str, sid: str, title: str | None, role: str, depth: int, st: _State) -> None:609 raw = chunk.strip()610 if not raw:611 return612 m = re.fullmatch(f"{CAP}(\\d+){CAP}", raw)613 if m:614 n0 = len(st.cites)615 cap = clean_inline(st.captions[int(m.group(1))], st)616 if cap:617 blocks.append(_Block(sid, title, "caption", depth, cap, ["caption"], st.cites[n0:]))618 return619 flags = []620 if raw.startswith(ITEM):621 flags.append("list_item")622 raw = raw.lstrip(ITEM).strip()623 raw = raw.replace(ITEM, " ")624 n0 = len(st.cites)625 text = clean_inline(raw, st)626 cites = st.cites[n0:]627 if not re.search(r"[A-Za-z]", TOKEN_RE.sub("", text)) and not TOKEN_RE.search(text):628 return629 if _is_display_only(text, st):630 prev = next((b for b in reversed(blocks) if b.role != "caption"), None)631 if prev is not None and prev.sec_id == sid:632 prev.text = f"{prev.text} {text}"633 prev.cites += cites634 if "has_disp_formula" not in prev.flags:635 prev.flags.append("has_disp_formula")636 return637 if any(st.math[int(t)][1] for t in TOKEN_RE.findall(text)):638 flags.append("has_disp_formula")639 blocks.append(_Block(sid, title, role, depth, text, flags, cites))640641642# ------------------------------------------------------------------------------------------ main643def _pre2018(pub_date: str | None) -> bool:644 if not pub_date:645 return False646 iso = pub_date + ("-01" if len(pub_date) == 7 else "-01-01" if len(pub_date) == 4 else "")647 try:648 return _dt.date.fromisoformat(iso) < _dt.date(2018, 1, 1)649 except ValueError:650 return False651652653def latex_body(main_text: str, st: _State) -> tuple[str | None, str, bool]:654 """Return (abstract_raw, body_raw, appendix_dropped) with math tokenised and floats replaced."""655 b = main_text.find("\\begin{document}")656 preamble, body = main_text[:b], main_text[b + len("\\begin{document}"):]657 e = body.find("\\end{document}")658 if e < 0:659 st.warn("no \\end{document}")660 else:661 body = body[:e]662 collect_macros(preamble, st)663 collect_macros(body, st)664 body = replace_commands(body, re.compile(r"\\def\s*\\[a-zA-Z]+[^{\n]*"), lambda m, o, r: " ")665 body = expand_macros(body, st, True)666 abstract, body = extract_abstract(body)667 if abstract is None:668 abstract, _ = extract_abstract(preamble)669 abstract = expand_macros(abstract, st, True) if abstract else None670 body, appendix = cut_tail(body, st)671 abstract = tokenize_math(abstract, st) if abstract else None672 body = tokenize_math(body, st)673 if brace_balance(body):674 st.warn(f"unbalanced braces in body ({brace_balance(body):+d})")675 body = _scan_dollars(expand_macros(body, st, False), st)676 abstract = _scan_dollars(expand_macros(abstract, st, False), st) if abstract else None677 body = structural_envs(extract_floats(body, st), st)678 abstract = structural_envs(abstract, st) if abstract else None679 return abstract, body, appendix680681682def parse_latex_text(main_text: str, stub: dict, warnings: list[str] | None = None) -> ParsedDocument:683 st = _State(warnings=list(warnings or []))684 abstract, body, appendix = latex_body(main_text, st)685 blocks: list[_Block] = []686 if abstract:687 for chunk in re.split(r"\n[ \t]*\n+", abstract):688 _add_chunk(blocks, chunk, "abstract", "Abstract", "abstract", 1, st)689 else:690 st.warn("no abstract environment found")691 blocks += blocks_from_sections(split_sections(body, bool(re.search(r"\\chapter\b", body)), st), st)692 paragraphs, parts, cursor = [], [], 0693 for i, bl in enumerate(blocks):694 text = restore_math(bl.text, st)695 pr = protect(text)696 cites = list(pr.citations) + [c for c in bl.cites if c not in pr.citations]697 flags = bl.flags + (["has_citation"] if cites else [])698 paragraphs.append(ParsedParagraph(bl.sec_id, bl.title, bl.role, bl.depth, i, text, cursor, cursor + len(text), cites, pr.math, pr.urls, pr.numbers, flags))699 parts.append(text + "\n\n")700 cursor += len(text) + 2701 words = sum(len(p.text.split()) for p in paragraphs)702 if words and st.residue / words > 0.02:703 st.warn(f"macro residue rate {st.residue / words:.1%} > 2%")704 if not paragraphs:705 st.warn("no paragraphs extracted")706 return _document(stub, main_text, "".join(parts), paragraphs, st)707708709def _document(stub: dict, main_text: str, body_text: str, paragraphs: list[ParsedParagraph], st: _State) -> ParsedDocument:710 lic = stub.get("license") or {}711 title = stub.get("title")712 if not title:713 m = re.search(r"\\title\s*(?:\[[^\]]*\])?\s*(?=\{)", main_text)714 g = read_group(main_text, m.end()) if m else None715 title = restore_math(clean_inline(g[0], st), st) if g else None716 pub_date = stub.get("pub_date")717 evidence = stub.get("pub_date_evidence")718 ids = {k: v for k, v in (stub.get("source_ids") or {}).items() if v}719 return ParsedDocument(720 title=title, authors=list(stub.get("authors") or []), author_group=stub.get("author_group"), venue=stub.get("venue"),721 publisher=stub.get("publisher"), doc_type=stub.get("doc_type") or "preprint", pub_date=pub_date,722 pub_year=stub.get("pub_year") or (int(pub_date[:4]) if pub_date else None), pub_date_evidence=evidence,723 pre2018_verified=_pre2018(pub_date) and evidence is not None, subjects=list(stub.get("subject_raw") or []), language=None,724 license_code=lic.get("code") or "UNKNOWN", license_uri=lic.get("uri"), license_evidence=lic.get("evidence"),725 body_text=body_text, paragraphs=paragraphs, warnings=st.warnings, article_type="preprint", ids=ids,726 )727728729def parse_latex_archive(path: Path, stub: dict) -> ParsedDocument:730 """Parse an arXiv e-print (``.tar.gz``, ``.gz``, ``.tar`` or ``.tex``) into a ``ParsedDocument``."""731 st = _State()732 with tempfile.TemporaryDirectory(prefix="htr-latex-") as tmp:733 root = Path(tmp)734 extract_archive(Path(path), root, st)735 files = [p for p in root.rglob("*") if p.is_file()]736 main = find_main(files, st)737 text = strip_comments(read_text(main, st))738 text = resolve_inputs(text, main.parent, root, st, 0, {str(main.resolve())})739 return parse_latex_text(text, stub, st.warnings)740