#!/usr/bin/env python3
"""Build evaluation fixtures (protocol heldout_v1) from held-out documents: Markdown 'manuscripts' with verified material only
(abstract, headings, section openings, table captions/notes, figure captions, references) — the published introduction is withheld
for the draft task. Also builds the out-of-domain fixture pointer (WP29). Fixture texts contain copyrighted excerpts → gitignored."""
import json, random, re, sys, shutil
from pathlib import Path
import pandas as pd
sys.path.insert(0, "src")
from jof250 import config, segmentation
from jof250.provenance import stamp
random.seed(45)
R = config.ROOT; FX = R / "experiments" / "fixtures" / "heldout_v1"; FX.mkdir(parents=True, exist_ok=True)
sp = json.loads((R / "data" / "splits" / "heldout_v1.json").read_text()); held = sp["heldout"]
ann = pd.read_parquet(R / "data" / "annotations" / "annotations_table.parquet"); ann = ann[ann.schema_valid == True]
summ = pd.read_parquet(R / "data" / "derived" / "parse_summary.parquet")
cand = ann[ann.doc_id.isin(held) & (ann.genre == "empirical")].merge(summ[["doc_id", "n_tables", "n_references", "confidence"]], on="doc_id")
cand = cand[(cand.n_tables >= 6) & (cand.confidence >= 0.9)]
# two different subfields, prefer the two most common
picks = []
for sf in cand.subfield.value_counts().index:
    pool = sorted(cand[cand.subfield == sf].doc_id.tolist())
    if pool: picks.append(random.choice(pool))
    if len(picks) == 2: break
def md_for(doc_id: str, withhold_intro: bool) -> tuple[str, dict]:
    d = segmentation.load_parsed(doc_id); m = d["metadata"]
    L = [f"# {m['title']}", "", f"*Journal of Finance {int(m['year'])}, vol. {int(m['volume'])}({int(m['issue'])}), pp. {m['pages']}; DOI {m['doi']}. Authors: {m['bib_authors']}.*", "", "## Abstract", "", d["abstract"] or ""]
    lit_paras = []
    for s in d["sections"]:
        role = segmentation.section_role(s["heading"], s.get("is_appendix", False), s.get("is_intro", False)); ps = segmentation.paragraphs(s)
        if role == "introduction" and withhold_intro:
            # keep only citation-bearing sentences from the intro as a degraded literature list (for rewrite_literature)
            for p in ps:
                for sent in re.split(r"(?<=[.!?])\s+", p["text"]):
                    if re.search(r"\(\d{4}[a-z]?\)", sent): lit_paras.append(sent)
            L += ["", "## Introduction", "", "*(withheld)*"]; continue
        if role == "appendix": continue
        head = f"{s.get('number') + '. ' if s.get('number') else ''}{s['heading']}"; L += ["", f"## {head}", ""]
        if s["subsections"]: L.append("Subsections: " + "; ".join(x["heading"] for x in s["subsections"]) + "\n")
        for p in ps[:3]: L.append(p["text"] + "\n")
        if len(ps) > 3: L.append(f"*({len(ps) - 3} further paragraphs omitted)*\n")
    L += ["", "## Tables", ""] + [f"**{t['label']}** — {t['title'] or ''}\n\nNotes: {t['notes_excerpt'] or '(none captured)'}\n" for t in d["tables"]]
    L += ["", "## Figures", ""] + [f"**{f['label']}** — {f['caption']}\n" for f in d["figures"]]
    L += ["", "## References", ""] + [f"- {r['text']}" for r in d["references"]]
    meta = {"doc_id": doc_id, "title": m["title"], "doi": m["doi"], "year": m["year"], "n_tables": len(d["tables"]), "n_refs": len(d["references"]), "degraded_literature": " ".join(lit_paras)[:4000],
            "published_intro_words": next((s["n_words"] for s in d["sections"] if s.get("is_intro")), None), "sha256": d["source_sha256"]}
    return "\n".join(L), meta
manifest = {"protocol": "heldout_v1", "built": stamp(), "finance_docs": [], "out_of_domain": {"id": "wp29", "path": str(Path("~/Desktop/UQO/Recherche/Recherche-UQO/methodes/autonomous-research-claude-code").expanduser()), "kind": "latex", "note": "UQO WP29, author's own manuscript, methods/CS-flavoured; verified results"}}
for did in picks:
    dd = FX / did; dd.mkdir(exist_ok=True)
    txt, meta = md_for(did, withhold_intro=True); (dd / "manuscript.md").write_text(txt)
    full, _ = md_for(did, withhold_intro=False); (dd / "manuscript_with_intro.md").write_text(full)
    d = segmentation.load_parsed(did); intro = next((s for s in d["sections"] if s.get("is_intro")), None)
    (dd / "published_introduction.txt").write_text("\n\n".join(p["text"] for p in segmentation.paragraphs(intro)) if intro else "")
    (dd / "meta.json").write_text(json.dumps(meta, indent=1)); manifest["finance_docs"].append(meta)
(FX / "manifest.json").write_text(json.dumps(manifest, indent=1)); print(json.dumps({k: v for k, v in manifest.items() if k != "finance_docs"}, indent=1)); print([(m["doc_id"], m["title"][:50], m["n_tables"]) for m in manifest["finance_docs"]])