#!/usr/bin/env python3
"""Parser QA: independent cross-check of the PyMuPDF structural parser against regex counts on poppler `pdftotext -layout`
output for a stratified sample, plus title agreement with Crossref for ALL documents. Writes data/derived/parser_qa.{json,md}.
This is a second automated extraction, not a human gold standard (see docs/LIMITATIONS.md)."""
import json, re, subprocess, sys, random, unicodedata
from pathlib import Path
import pandas as pd
sys.path.insert(0, "src")
from jof250 import config
from jof250.provenance import stamp
from rapidfuzz import fuzz

random.seed(45)
man = pd.read_parquet("data/manifests/corpus_manifest.parquet"); inc = man[man.decision == "include"]
summ = pd.read_parquet("data/derived/parse_summary.parquet")
# stratified sample: 5 per year + all non-article types (cap 6)
sample = []
for y in sorted(inc.bib_year.unique()):
    sample += random.sample(inc[(inc.bib_year == y) & (inc.article_type == "article")].doc_id.tolist(), 5)
sample += inc[inc.article_type != "article"].doc_id.tolist()[:6]
ROMAN = r"(?:I|II|III|IV|V|VI|VII|VIII|IX|X|XI|XII)"
rows = []
for did in sample:
    r = inc[inc.doc_id == did].iloc[0]; pdf = config.corpus_root() / r.relpath
    txt = subprocess.run(["pdftotext", "-layout", str(pdf), "-"], capture_output=True, text=True).stdout
    txt = unicodedata.normalize("NFKC", txt)
    lines = [l.strip() for l in txt.splitlines()]
    body = "\n".join(lines)
    ref_i = next((i for i, l in enumerate(lines) if l == "REFERENCES"), None)
    pre = "\n".join(lines[:ref_i]) if ref_i else body
    post = lines[ref_i + 1:] if ref_i else []
    n_sec = len([l for l in lines[:ref_i or len(lines)] if re.match(rf"^{ROMAN}\.\s+[A-Z]", l) and len(l) < 110 and not l.endswith(",")])
    n_tab = len(set(re.findall(r"^\s*(Table [IVXLC]+)\s*$", pre, re.M)))
    n_fig = len(set(re.findall(r"^\s*(Figure \d+)\.\s", pre, re.M)))
    n_eq = len(re.findall(r"\((\d{1,3})\)\s*$", pre, re.M))
    # references: entries start with "Surname, " ... year — count lines in refs block starting at column 0 with a comma-separated author then a 4-digit year
    n_ref = len([l for l in post if re.match(r"^[A-Z][^\n]{2,}?,\s.*\b(19|20)\d{2}[a-z]?,", l) and not l.startswith("The Journal of Finance")]) if post else 0
    has_abs = bool(re.search(r"^\s*ABSTRACT\s*$", pre, re.M))
    j = json.loads((config.derived_dir() / "parsed" / f"{did}.json").read_text())
    rows.append({"doc_id": did, "type": r.article_type, "year": int(r.bib_year),
                 "sections_parser": j["stats"]["n_sections"], "sections_regex": n_sec,
                 "tables_parser": j["stats"]["n_tables"], "tables_regex": n_tab,
                 "figures_parser": j["stats"]["n_figures"], "figures_regex": n_fig,
                 "equations_parser": j["stats"]["n_equations"], "equations_regex": n_eq,
                 "references_parser": j["stats"]["n_references"], "references_regex": n_ref,
                 "abstract_parser": bool(j["abstract"]), "abstract_regex": has_abs})
df = pd.DataFrame(rows)
def agree(a, b, tol=0): return float(((df[a] - df[b]).abs() <= tol).mean())
metrics = {"n_sample": len(df),
           "sections_exact": agree("sections_parser", "sections_regex"), "tables_exact": agree("tables_parser", "tables_regex"), "tables_within1": agree("tables_parser", "tables_regex", 1),
           "figures_exact": agree("figures_parser", "figures_regex"), "equations_exact": agree("equations_parser", "equations_regex"), "equations_within10pct": float(((df.equations_parser - df.equations_regex).abs() <= (0.1 * df.equations_regex).clip(lower=1)).mean()),
           "references_within10pct": float(((df.references_parser - df.references_regex).abs() <= (0.1 * df.references_regex).clip(lower=2)).mean()),
           "abstract_agree": float((df.abstract_parser == df.abstract_regex).mean())}
# title agreement on all docs
titles = []
for r in inc.itertuples():
    j = json.loads((config.derived_dir() / "parsed" / f"{r.doc_id}.json").read_text())
    titles.append(fuzz.ratio((j["metadata"]["title"] or "").lower(), str(r.crossref_title).lower()))
metrics["title_vs_crossref_ge90_all_docs"] = float(pd.Series(titles).ge(90).mean()); metrics["title_n_all_docs"] = len(titles)
out = {"generated": stamp(), "metrics": metrics, "sample": rows}
Path("data/derived/parser_qa.json").write_text(json.dumps(out, indent=1))
md = ["# Parser QA (automated cross-check)", "", f"Generated {out['generated']['timestamp']} (code {out['generated']['code_version']}). Sample: {len(df)} documents (5 random articles per year, seed 45, + all non-article types). Reference extraction = regex on `pdftotext -layout`; this is a cross-parser agreement check, **not** a human gold standard.", "", "| metric | value |", "|---|---|"]
md += [f"| {k} | {v:.3f} |" if isinstance(v, float) else f"| {k} | {v} |" for k, v in metrics.items()]
md += ["", "## Per-document comparison", "", "| doc | type | sec P/R | tab P/R | fig P/R | eq P/R | ref P/R | abs P/R |", "|---|---|---|---|---|---|---|---|"]
md += [f"| {r.doc_id} | {r.type} | {r.sections_parser}/{r.sections_regex} | {r.tables_parser}/{r.tables_regex} | {r.figures_parser}/{r.figures_regex} | {r.equations_parser}/{r.equations_regex} | {r.references_parser}/{r.references_regex} | {int(r.abstract_parser)}/{int(r.abstract_regex)} |" for r in df.itertuples()]
Path("data/derived/parser_qa.md").write_text("\n".join(md) + "\n")
print(json.dumps(metrics, indent=1)); print(df.to_string())