1"""Phase 2 — structured extraction of JoF PDFs with PyMuPDF (layout heuristics tuned on the JoF LaTeX template).23Output: data/derived/parsed/<doc_id>.json (private: contains full text) with per-component confidence flags.4"""5from __future__ import annotations6import json, re, unicodedata, statistics7from pathlib import Path8from dataclasses import dataclass, field, asdict9import pandas as pd10import pymupdf11from .provenance import sha256_file, stamp, Ledger, now_iso12from . import config1314PARSER_VERSION = "pymupdf-jof-heuristics-1.3"15ROMAN = r"(?:I|II|III|IV|V|VI|VII|VIII|IX|X|XI|XII|XIII|XIV|XV)"16SEC_RE = re.compile(rf"^{ROMAN}\.\s+\S")17SUB_RE = re.compile(r"^[A-Z]\.(\d+\.)?\s+[A-Z0-9(“\"']")18EQ_TAG = re.compile(r"\((\d{1,3}[a-z]?|[A-Z]\d{1,2}|IA\.?\d+)\)\s*$")19TABLE_RE = re.compile(r"^Table\s+([IVXLC]+|[A-Z]?\d+|A\.?[IVX]+)(—Continued)?\s*$")20FIG_RE = re.compile(r"^Figure\s+(\d+|[A-Z]\.?\d+)\.\s")21CITE_RE = re.compile(r"\b([A-Z][A-Za-z'’\-]+(?:,? (?:and |& )?[A-Z][A-Za-z'’\-]+)*(?: et al\.)?)\s*\((\d{4}[a-z]?(?:,\s*\d{4}[a-z]?)*)\)")2223@dataclass24class Line:25 page: int; x0: float; y0: float; x1: float; y1: float; font: str; size: float; text: str; bold: bool; italic: bool; mathfont: bool; bold_start: bool = False2627def norm(s: str) -> str:28 s = unicodedata.normalize("NFKC", s)29 s = "".join(ch for ch in s if ch.isprintable() or ch in "\t ")30 return s.replace("−", "-").replace("\xa0", " ")3132def page_lines(doc: pymupdf.Document, pno: int, rotated: list | None = None) -> list[Line]:33 page = doc[pno]; out = []34 for b in page.get_text("dict")["blocks"]:35 if b["type"] != 0: continue36 for l in b["lines"]:37 spans = [s for s in l["spans"] if s["text"].strip()]38 if not spans: continue39 if abs(l.get("dir", (1, 0))[0]) < 0.5: # rotated (landscape table) line40 if rotated is not None:41 main = max(spans, key=lambda s: len(s["text"]))42 rotated.append(Line(pno + 1, *l["bbox"], main["font"], round(main["size"], 1), norm("".join(s["text"] for s in l["spans"])).strip(), "Bold" in main["font"], "Ital" in main["font"], False))43 continue44 text = norm("".join(s["text"] for s in l["spans"])).strip()45 if not text: continue46 # dominant span by chars47 main = max(spans, key=lambda s: len(s["text"]))48 font = main["font"]; size = round(main["size"], 1)49 if "Times" in font and size < 5: continue # Wiley download watermark in margin50 x0, y0, x1, y1 = l["bbox"]51 if y1 - y0 > 3 * max(size, 1) and (x1 - x0) < 15: continue # vertical (rotated) text fragments52 out.append(Line(pno + 1, x0, y0, x1, y1, font, size, text, "Bold" in font, "Ital" in font, font.startswith(("MT", "RMT", "CM", "MS")), "Bold" in spans[0]["font"]))53 return out5455def is_header(l: Line) -> bool:56 return l.y0 < 565758def body_size(lines: list[Line]) -> float:59 sizes = [l.size for l in lines if "Schlbk-Roman" in l.font or "Roman" in l.font]60 try: return statistics.mode([s for s in sizes if s >= 9]) if any(s >= 9 for s in sizes) else 10.061 except statistics.StatisticsError: return 10.06263def parse_pdf(path: Path, doc_id: str, meta: dict) -> dict:64 doc = pymupdf.open(path)65 all_lines: list[Line] = []; rotated: list[Line] = []66 for p in range(doc.page_count):67 all_lines += [l for l in page_lines(doc, p, rotated) if not is_header(l)]68 BS = body_size(all_lines)69 pw = doc[0].rect.width70 # ---------- front matter ----------71 first = [l for l in all_lines if l.page <= 3]72 title_lines = [l for l in first if l.bold and l.size >= BS + 2.5]73 title = " ".join(l.text for l in title_lines).strip() or None74 title_page = title_lines[0].page if title_lines else 175 # authors: roman ~9pt lines between title and ABSTRACT76 abs_idx = next((i for i, l in enumerate(all_lines) if l.text.upper().startswith("ABSTRACT") and l.page <= 3), None)77 authors = None; abstract = None; abstract_conf = 0.078 if abs_idx is not None:79 t_end = all_lines.index(title_lines[-1]) if title_lines else 080 a_lines = [l for l in all_lines[t_end + 1:abs_idx] if l.size < BS and l.size >= BS - 1.5 and not l.bold]81 authors = " ".join(l.text for l in a_lines).replace("∗", "").strip() or None82 ab = []83 for l in all_lines[abs_idx + 1:]:84 if l.size >= BS - 0.5 or l.page > title_page + 1: break85 if l.size <= BS - 1.5 and not l.bold: ab.append(l.text)86 abstract = " ".join(ab).strip() or None87 abstract_conf = 1.0 if abstract and 400 < len(abstract) < 2500 else (0.5 if abstract else 0.0)88 # ---------- structural pass ----------89 sections = []; cur = {"number": None, "heading": "Introduction (unnumbered)", "level": 0, "page": 1, "paragraphs": [], "subsections": [], "_lines": [], "is_intro": True}90 equations, tables, figures, footnotes, refs_lines = [], [], [], [], []91 in_refs = False; in_appendix = False; refs_page = None; appendix_page = None; supporting_page = None92 in_table_block = False; table_block_page = None93 fig_cap_y = {}94 for l in all_lines:95 if (l.bold or l.bold_start) and l.size <= BS - 1.5 and FIG_RE.match(l.text): fig_cap_y[l.page] = min(l.y0, fig_cap_y.get(l.page, 1e9))96 i = 0; n = len(all_lines)97 body_left = statistics.median([l.x0 for l in all_lines if l.size == BS and not l.italic]) if any(l.size == BS for l in all_lines) else 6198 while i < n:99 l = all_lines[i]; t = l.text100 centered = abs((l.x0 + l.x1) / 2 - pw / 2) < 40101 # page number lines (digits only, near bottom)102 if re.fullmatch(r"\d{1,4}", t) and l.y0 > 600: i += 1; continue103 # references104 if l.bold and l.size >= BS - 0.5 and t.upper().strip() in ("REFERENCES", "REFERENCE"):105 in_refs = True; refs_page = l.page; in_table_block = False; i += 1; continue106 if l.bold and l.size >= BS - 0.5 and t.startswith("Supporting Information"):107 supporting_page = l.page; in_refs = False; i += 1; continue108 if in_refs:109 if l.bold and l.size >= BS - 0.5 and (t.lower().startswith("appendix") or SEC_RE.match(t)):110 in_refs = False # appendix after refs111 else:112 if l.size <= BS - 1.5 or l.size == BS: refs_lines.append(l)113 i += 1; continue114 # table caption115 if l.bold and l.size <= BS - 1.5 and TABLE_RE.match(t):116 m = TABLE_RE.match(t)117 if m.group(2): in_table_block = True; table_block_page = l.page; i += 1; continue118 ttl = []; j = i + 1119 while j < n and all_lines[j].bold and all_lines[j].size >= BS - 0.5 and all_lines[j].page == l.page: ttl.append(all_lines[j].text); j += 1120 notes = []; k = j121 while k < n and all_lines[k].page == l.page and all_lines[k].size <= BS - 1.5 and not all_lines[k].bold and len(notes) < 25:122 notes.append(all_lines[k].text); k += 1123 if sum(len(x) for x in notes) > 2500: break124 tables.append({"label": f"Table {m.group(1)}", "title": " ".join(ttl) or None, "page": l.page, "notes_excerpt": " ".join(notes)[:1500], "rotated": l.x1 - l.x0 < 20, "section": cur["heading"]})125 in_table_block = True; table_block_page = l.page; i = k; continue126 if in_table_block and l.page != table_block_page: in_table_block = False127 # figure caption128 if (l.bold or l.bold_start) and l.size <= BS - 1.5 and FIG_RE.match(t):129 cap = [t]; j = i + 1130 while j < n and all_lines[j].page == l.page and all_lines[j].size <= BS - 1.5 and not FIG_RE.match(all_lines[j].text) and len(cap) < 15: cap.append(all_lines[j].text); j += 1131 figures.append({"label": FIG_RE.match(t).group(0).strip().rstrip("."), "caption": " ".join(cap)[:1500], "page": l.page, "section": cur["heading"]})132 i = j; continue133 # section heading (bold, body size, centered or starts with roman numeral)134 if l.bold and l.size >= BS - 0.5 and l.size < BS + 2 and (SEC_RE.match(t) or centered) and not TABLE_RE.match(t) and len(t) < 120:135 head = [t]; j = i + 1136 while j < n and all_lines[j].bold and all_lines[j].size >= BS - 0.5 and all_lines[j].page == l.page and not SEC_RE.match(all_lines[j].text) and abs(all_lines[j].y0 - all_lines[j-1].y1) < 8: head.append(all_lines[j].text); j += 1137 h = " ".join(head)138 m = re.match(rf"^({ROMAN})\.\s+(.*)$", h)139 is_app = h.lower().startswith("appendix") or (cur["heading"].lower().startswith("appendix") and m is None)140 if h.lower().startswith("appendix"): in_appendix = True; appendix_page = appendix_page or l.page141 sections.append(cur)142 cur = {"number": m.group(1) if m else None, "heading": m.group(2) if m else h, "level": 1, "page": l.page, "paragraphs": [], "subsections": [], "_lines": [], "is_appendix": in_appendix or is_app}143 i = j; continue144 # subsection heading (italic, body size, short, pattern "A. Title")145 if l.italic and l.size >= BS - 0.5 and SUB_RE.match(t) and len(t) < 110 and not t.endswith((",", ";")) and (i + 1 < n and (all_lines[i+1].size <= BS - 1.5 or not all_lines[i+1].italic or all_lines[i+1].y0 - l.y1 > 3)):146 cur["subsections"].append({"heading": t, "page": l.page, "para_index": len(cur["paragraphs"]) + (1 if cur["_lines"] else 0)})147 cur["_lines"].append(("SUBHEAD", t)); i += 1; continue148 # footnotes: small (BS-2) lines in lower part preceded by tiny number, or small lines once a footnote started on this page149 if l.size <= BS - 1.5 and not in_table_block:150 if l.page in fig_cap_y and l.y0 < fig_cap_y[l.page]: i += 1; continue # text inside a figure151 if len(re.sub(r"[^A-Za-z]", "", t)) < 3 and not t.startswith("∗"): i += 1; continue # axis ticks / stray numbers152 if l.page <= 3 and abstract and l.y0 < 300 and l.size <= BS - 1.5 and l.x0 > body_left + 15: i += 1; continue # abstract already captured153 if l.size <= BS - 3.5 or re.match(r"^\d{1,3}\s", t) or (not footnotes) or footnotes[-1]["page"] != l.page or t.startswith("∗"):154 footnotes.append({"page": l.page, "text": t, "n_lines": 1})155 else:156 footnotes[-1]["text"] = join_lines([footnotes[-1]["text"], t]); footnotes[-1]["n_lines"] += 1157 i += 1; continue158 # equations: numbered tag at right margin159 if EQ_TAG.search(t) and l.x1 > pw * 0.8 and l.size >= BS - 0.5 and (len(t) < 6 or l.italic or l.mathfont or re.search(r"[=+−×∑∫]", t)):160 tag = EQ_TAG.search(t).group(1)161 band = [x for x in all_lines[max(0, i-6):i] if x.page == l.page and abs(x.y0 - l.y0) < 14 and x is not l]162 body = " ".join(x.text for x in band) + (" " + t[:EQ_TAG.search(t).start()].strip() if len(t) > 6 else "")163 equations.append({"tag": tag, "page": l.page, "text_excerpt": body.strip()[:300], "section": cur["heading"], "confidence": 0.9 if body.strip() else 0.5})164 i += 1; continue165 if l.mathfont: cur["_lines"].append(("MATH", t)); i += 1; continue166 # body line167 if l.size >= BS - 0.5 and l.size <= BS + 0.5:168 cur["_lines"].append(("P_NEW" if l.x0 > body_left + 6 else "P", t, l.page))169 i += 1170 sections.append(cur)171 # rotated (landscape) tables172 for idx, l in enumerate(rotated):173 m = TABLE_RE.match(l.text)174 if l.bold and m and not m.group(2):175 ttl = [x.text for x in rotated[idx+1:idx+4] if x.bold and x.page == l.page and x.size >= BS - 0.5]176 notes = [x.text for x in rotated[idx+1:idx+30] if x.page == l.page and not x.bold and x.size <= BS - 1.5]177 tables.append({"label": f"Table {m.group(1)}", "title": " ".join(ttl) or None, "page": l.page, "notes_excerpt": " ".join(notes)[:1500], "rotated": True, "section": None})178 tables.sort(key=lambda t: t["page"])179 # ---------- paragraphs from lines ----------180 def build_paras(sec):181 paras = []; buf = []; pg = sec["page"]182 for item in sec["_lines"]:183 kind = item[0]184 if kind == "SUBHEAD":185 if buf: paras.append({"text": join_lines(buf), "page": pg}); buf = []186 paras.append({"text": item[1], "page": pg, "kind": "subheading"}); continue187 if kind == "MATH":188 if buf: buf.append(item[1]); continue189 continue190 if kind == "P_NEW" and buf:191 paras.append({"text": join_lines(buf), "page": pg}); buf = []192 if not buf: pg = item[2]193 buf.append(item[1])194 if buf: paras.append({"text": join_lines(buf), "page": pg})195 sec["paragraphs"] = [p for p in paras if len(p["text"]) > 1]196 del sec["_lines"]197 sec["n_words"] = sum(len(p["text"].split()) for p in sec["paragraphs"] if p.get("kind") != "subheading")198 return sec199 sections = [build_paras(s) for s in sections]200 sections = [s for s in sections if s["paragraphs"] or not s.get("is_intro")]201 # ---------- references ----------202 references = split_references(refs_lines)203 # ---------- citations in text ----------204 body_text = "\n".join(p["text"] for s in sections for p in s["paragraphs"] if p.get("kind") != "subheading")205 cites = CITE_RE.findall(body_text)206 cite_strings = [f"{a} ({y})" for a, y in cites]207 # ---------- roles & confidence ----------208 numbered = [s for s in sections if s.get("number")]209 roman_seq = [s["number"] for s in numbered]210 conf = {211 "title": 1.0 if title else 0.0, "authors": 1.0 if authors else 0.0, "abstract": abstract_conf,212 "sections": 1.0 if len(numbered) >= 3 and roman_seq == sorted(roman_seq, key=roman_val) else (0.6 if len(numbered) >= 2 else 0.2),213 "references": 1.0 if len(references) >= 10 else (0.5 if references else 0.0),214 "equations": 0.9 if equations else 0.7, "tables": 0.9 if tables else 0.7, "figures": 0.9,215 }216 conf["overall"] = round(sum(conf.values()) / len(conf), 3)217 out = {218 "doc_id": doc_id, "source_sha256": sha256_file(path), "source_relpath": meta.get("relpath"), "parser": PARSER_VERSION, "parsed_at": now_iso(),219 "pages": doc.page_count, "body_font_size": BS,220 "metadata": {"title": title, "authors": authors, "doi": meta.get("bib_doi"), "year": meta.get("bib_year"), "volume": meta.get("bib_volume"), "issue": meta.get("bib_issue"), "pages": meta.get("bib_pages"), "article_type": meta.get("article_type"), "bib_title": meta.get("bib_title"), "bib_authors": meta.get("bib_authors")},221 "abstract": abstract, "sections": sections, "equations": equations, "tables": tables, "figures": figures, "footnotes": footnotes,222 "references": references, "citations_in_text": cite_strings,223 "markers": {"references_page": refs_page, "appendix_page": appendix_page, "supporting_information_page": supporting_page, "has_internet_appendix": bool(supporting_page) or "Internet Appendix" in body_text},224 "stats": {"n_sections": len(numbered), "n_subsections": sum(len(s["subsections"]) for s in sections), "n_paragraphs": sum(len(s["paragraphs"]) for s in sections), "n_words_body": sum(s["n_words"] for s in sections if not s.get("is_appendix")), "n_words_appendix": sum(s["n_words"] for s in sections if s.get("is_appendix")), "n_equations": len(equations), "n_tables": len(tables), "n_figures": len(figures), "n_footnotes": len(footnotes), "n_references": len(references), "n_citations_in_text": len(cite_strings), "abstract_words": len(abstract.split()) if abstract else 0},225 "confidence": conf,226 }227 doc.close()228 return out229230def join_lines(lines: list[str]) -> str:231 s = ""232 for x in lines:233 if s.endswith("-") and not s.endswith("--") and x[:1].islower(): s = s[:-1] + x234 else: s = (s + " " + x).strip()235 return s236237def split_references(lines: list[Line]) -> list[dict]:238 if not lines: return []239 # hanging indent: entry starts at the page's left margin (odd/even pages have different margins)240 left_by_page = {}241 for l in lines: left_by_page.setdefault(l.page, []).append(l.x0)242 left_by_page = {k: min(v) for k, v in left_by_page.items()}243 entries = []; buf = []; pg = None244 lines = [l for l in lines if not re.match(r"^DOI:\s*10\.", l.text)]245 for l in lines:246 if l.x0 <= left_by_page[l.page] + 3 and buf:247 entries.append({"text": join_lines(buf), "page": pg}); buf = []248 if not buf: pg = l.page249 buf.append(l.text)250 if buf: entries.append({"text": join_lines(buf), "page": pg})251 out = []252 for e in entries:253 m = re.match(r"^(.*?),\s*(\d{4}[a-z]?),\s*(.*)$", e["text"])254 e["authors"] = m.group(1) if m else None; e["year"] = m.group(2) if m else None; e["rest"] = m.group(3) if m else None255 e["doi"] = (re.search(r"10\.\d{4,9}/\S+", e["text"]) or [None])[0] if "10." in e["text"] else None256 out.append(e)257 return out258259def roman_val(r: str) -> int:260 v = {"I": 1, "V": 5, "X": 10}; t = 0261 for a, b in zip(r, r[1:] + " "):262 t += -v[a] if b != " " and v[a] < v[b] else v[a]263 return t264265def parse_corpus(limit: int | None = None, force: bool = False, only: str | None = None) -> dict:266 man = pd.read_parquet(config.manifests_dir() / "corpus_manifest.parquet")267 inc = man[man["decision"] == "include"]268 if only: inc = inc[inc["doc_id"] == only]269 if limit: inc = inc.head(limit)270 root = config.corpus_root(); outdir = config.derived_dir() / "parsed"; outdir.mkdir(parents=True, exist_ok=True)271 ledger = Ledger(config.derived_dir() / "parse_ledger.jsonl")272 done = ok = fail = 0; rows = []273 for r in inc.itertuples():274 outp = outdir / f"{r.doc_id}.json"275 if outp.exists() and not force:276 d = json.loads(outp.read_text()); rows.append(summary_row(d)); done += 1; continue277 try:278 d = parse_pdf(root / r.relpath, r.doc_id, r._asdict())279 outp.write_text(json.dumps(d, ensure_ascii=False)); rows.append(summary_row(d)); ok += 1280 ledger.append({"event": "parsed", "doc_id": r.doc_id, "confidence": d["confidence"]["overall"], "stats": d["stats"], **stamp()})281 except Exception as e:282 fail += 1; ledger.append({"event": "parse_error", "doc_id": r.doc_id, "error": repr(e)[:300], **stamp()})283 rows.append({"doc_id": r.doc_id, "error": repr(e)[:200]})284 df = pd.DataFrame(rows); df.to_parquet(config.derived_dir() / "parse_summary.parquet", index=False); df.to_csv(config.derived_dir() / "parse_summary.csv", index=False)285 return {"parsed_new": ok, "cached": done, "failed": fail, "n": len(rows), "mean_confidence": float(df["confidence"].mean()) if "confidence" in df else None}286287def summary_row(d: dict) -> dict:288 return {"doc_id": d["doc_id"], "year": d["metadata"]["year"], "article_type": d["metadata"]["article_type"], "pages": d["pages"], "confidence": d["confidence"]["overall"], **{f"conf_{k}": v for k, v in d["confidence"].items() if k != "overall"}, **d["stats"], "has_abstract": bool(d["abstract"]), "refs_page": d["markers"]["references_page"], "has_internet_appendix": d["markers"]["has_internet_appendix"]}289