1#!/usr/bin/env python32"""Parser QA: independent cross-check of the PyMuPDF structural parser against regex counts on poppler `pdftotext -layout`3output for a stratified sample, plus title agreement with Crossref for ALL documents. Writes data/derived/parser_qa.{json,md}.4This is a second automated extraction, not a human gold standard (see docs/LIMITATIONS.md)."""5import json, re, subprocess, sys, random, unicodedata6from pathlib import Path7import pandas as pd8sys.path.insert(0, "src")9from jof250 import config10from jof250.provenance import stamp11from rapidfuzz import fuzz1213random.seed(45)14man = pd.read_parquet("data/manifests/corpus_manifest.parquet"); inc = man[man.decision == "include"]15summ = pd.read_parquet("data/derived/parse_summary.parquet")16# stratified sample: 5 per year + all non-article types (cap 6)17sample = []18for y in sorted(inc.bib_year.unique()):19 sample += random.sample(inc[(inc.bib_year == y) & (inc.article_type == "article")].doc_id.tolist(), 5)20sample += inc[inc.article_type != "article"].doc_id.tolist()[:6]21ROMAN = r"(?:I|II|III|IV|V|VI|VII|VIII|IX|X|XI|XII)"22rows = []23for did in sample:24 r = inc[inc.doc_id == did].iloc[0]; pdf = config.corpus_root() / r.relpath25 txt = subprocess.run(["pdftotext", "-layout", str(pdf), "-"], capture_output=True, text=True).stdout26 txt = unicodedata.normalize("NFKC", txt)27 lines = [l.strip() for l in txt.splitlines()]28 body = "\n".join(lines)29 ref_i = next((i for i, l in enumerate(lines) if l == "REFERENCES"), None)30 pre = "\n".join(lines[:ref_i]) if ref_i else body31 post = lines[ref_i + 1:] if ref_i else []32 n_sec = len([l for l in lines[:ref_i or len(lines)] if re.match(rf"^{ROMAN}\.\s+[A-Z]", l) and len(l) < 110 and not l.endswith(",")])33 n_tab = len(set(re.findall(r"^\s*(Table [IVXLC]+)\s*$", pre, re.M)))34 n_fig = len(set(re.findall(r"^\s*(Figure \d+)\.\s", pre, re.M)))35 n_eq = len(re.findall(r"\((\d{1,3})\)\s*$", pre, re.M))36 # references: entries start with "Surname, " ... year — count lines in refs block starting at column 0 with a comma-separated author then a 4-digit year37 n_ref = len([l for l in post if re.match(r"^[A-Z][^\n]{2,}?,\s.*\b(19|20)\d{2}[a-z]?,", l) and not l.startswith("The Journal of Finance")]) if post else 038 has_abs = bool(re.search(r"^\s*ABSTRACT\s*$", pre, re.M))39 j = json.loads((config.derived_dir() / "parsed" / f"{did}.json").read_text())40 rows.append({"doc_id": did, "type": r.article_type, "year": int(r.bib_year),41 "sections_parser": j["stats"]["n_sections"], "sections_regex": n_sec,42 "tables_parser": j["stats"]["n_tables"], "tables_regex": n_tab,43 "figures_parser": j["stats"]["n_figures"], "figures_regex": n_fig,44 "equations_parser": j["stats"]["n_equations"], "equations_regex": n_eq,45 "references_parser": j["stats"]["n_references"], "references_regex": n_ref,46 "abstract_parser": bool(j["abstract"]), "abstract_regex": has_abs})47df = pd.DataFrame(rows)48def agree(a, b, tol=0): return float(((df[a] - df[b]).abs() <= tol).mean())49metrics = {"n_sample": len(df),50 "sections_exact": agree("sections_parser", "sections_regex"), "tables_exact": agree("tables_parser", "tables_regex"), "tables_within1": agree("tables_parser", "tables_regex", 1),51 "figures_exact": agree("figures_parser", "figures_regex"), "equations_exact": agree("equations_parser", "equations_regex"), "equations_within10pct": float(((df.equations_parser - df.equations_regex).abs() <= (0.1 * df.equations_regex).clip(lower=1)).mean()),52 "references_within10pct": float(((df.references_parser - df.references_regex).abs() <= (0.1 * df.references_regex).clip(lower=2)).mean()),53 "abstract_agree": float((df.abstract_parser == df.abstract_regex).mean())}54# title agreement on all docs55titles = []56for r in inc.itertuples():57 j = json.loads((config.derived_dir() / "parsed" / f"{r.doc_id}.json").read_text())58 titles.append(fuzz.ratio((j["metadata"]["title"] or "").lower(), str(r.crossref_title).lower()))59metrics["title_vs_crossref_ge90_all_docs"] = float(pd.Series(titles).ge(90).mean()); metrics["title_n_all_docs"] = len(titles)60out = {"generated": stamp(), "metrics": metrics, "sample": rows}61Path("data/derived/parser_qa.json").write_text(json.dumps(out, indent=1))62md = ["# Parser QA (automated cross-check)", "", f"Generated {out['generated']['timestamp']} (code {out['generated']['code_version']}). Sample: {len(df)} documents (5 random articles per year, seed 45, + all non-article types). Reference extraction = regex on `pdftotext -layout`; this is a cross-parser agreement check, **not** a human gold standard.", "", "| metric | value |", "|---|---|"]63md += [f"| {k} | {v:.3f} |" if isinstance(v, float) else f"| {k} | {v} |" for k, v in metrics.items()]64md += ["", "## Per-document comparison", "", "| doc | type | sec P/R | tab P/R | fig P/R | eq P/R | ref P/R | abs P/R |", "|---|---|---|---|---|---|---|---|"]65md += [f"| {r.doc_id} | {r.type} | {r.sections_parser}/{r.sections_regex} | {r.tables_parser}/{r.tables_regex} | {r.figures_parser}/{r.figures_regex} | {r.equations_parser}/{r.equations_regex} | {r.references_parser}/{r.references_regex} | {int(r.abstract_parser)}/{int(r.abstract_regex)} |" for r in df.itertuples()]66Path("data/derived/parser_qa.md").write_text("\n".join(md) + "\n")67print(json.dumps(metrics, indent=1)); print(df.to_string())68