1#!/usr/bin/env python32"""Build evaluation fixtures (protocol heldout_v1) from held-out documents: Markdown 'manuscripts' with verified material only3(abstract, headings, section openings, table captions/notes, figure captions, references) — the published introduction is withheld4for the draft task. Also builds the out-of-domain fixture pointer (WP29). Fixture texts contain copyrighted excerpts → gitignored."""5import json, random, re, sys, shutil6from pathlib import Path7import pandas as pd8sys.path.insert(0, "src")9from jof250 import config, segmentation10from jof250.provenance import stamp11random.seed(45)12R = config.ROOT; FX = R / "experiments" / "fixtures" / "heldout_v1"; FX.mkdir(parents=True, exist_ok=True)13sp = json.loads((R / "data" / "splits" / "heldout_v1.json").read_text()); held = sp["heldout"]14ann = pd.read_parquet(R / "data" / "annotations" / "annotations_table.parquet"); ann = ann[ann.schema_valid == True]15summ = pd.read_parquet(R / "data" / "derived" / "parse_summary.parquet")16cand = ann[ann.doc_id.isin(held) & (ann.genre == "empirical")].merge(summ[["doc_id", "n_tables", "n_references", "confidence"]], on="doc_id")17cand = cand[(cand.n_tables >= 6) & (cand.confidence >= 0.9)]18# two different subfields, prefer the two most common19picks = []20for sf in cand.subfield.value_counts().index:21 pool = sorted(cand[cand.subfield == sf].doc_id.tolist())22 if pool: picks.append(random.choice(pool))23 if len(picks) == 2: break24def md_for(doc_id: str, withhold_intro: bool) -> tuple[str, dict]:25 d = segmentation.load_parsed(doc_id); m = d["metadata"]26 L = [f"# {m['title']}", "", f"*Journal of Finance {int(m['year'])}, vol. {int(m['volume'])}({int(m['issue'])}), pp. {m['pages']}; DOI {m['doi']}. Authors: {m['bib_authors']}.*", "", "## Abstract", "", d["abstract"] or ""]27 lit_paras = []28 for s in d["sections"]:29 role = segmentation.section_role(s["heading"], s.get("is_appendix", False), s.get("is_intro", False)); ps = segmentation.paragraphs(s)30 if role == "introduction" and withhold_intro:31 # keep only citation-bearing sentences from the intro as a degraded literature list (for rewrite_literature)32 for p in ps:33 for sent in re.split(r"(?<=[.!?])\s+", p["text"]):34 if re.search(r"\(\d{4}[a-z]?\)", sent): lit_paras.append(sent)35 L += ["", "## Introduction", "", "*(withheld)*"]; continue36 if role == "appendix": continue37 head = f"{s.get('number') + '. ' if s.get('number') else ''}{s['heading']}"; L += ["", f"## {head}", ""]38 if s["subsections"]: L.append("Subsections: " + "; ".join(x["heading"] for x in s["subsections"]) + "\n")39 for p in ps[:3]: L.append(p["text"] + "\n")40 if len(ps) > 3: L.append(f"*({len(ps) - 3} further paragraphs omitted)*\n")41 L += ["", "## Tables", ""] + [f"**{t['label']}** — {t['title'] or ''}\n\nNotes: {t['notes_excerpt'] or '(none captured)'}\n" for t in d["tables"]]42 L += ["", "## Figures", ""] + [f"**{f['label']}** — {f['caption']}\n" for f in d["figures"]]43 L += ["", "## References", ""] + [f"- {r['text']}" for r in d["references"]]44 meta = {"doc_id": doc_id, "title": m["title"], "doi": m["doi"], "year": m["year"], "n_tables": len(d["tables"]), "n_refs": len(d["references"]), "degraded_literature": " ".join(lit_paras)[:4000],45 "published_intro_words": next((s["n_words"] for s in d["sections"] if s.get("is_intro")), None), "sha256": d["source_sha256"]}46 return "\n".join(L), meta47manifest = {"protocol": "heldout_v1", "built": stamp(), "finance_docs": [], "out_of_domain": {"id": "wp29", "path": str(Path("~/Desktop/UQO/Recherche/Recherche-UQO/methodes/autonomous-research-claude-code").expanduser()), "kind": "latex", "note": "UQO WP29, author's own manuscript, methods/CS-flavoured; verified results"}}48for did in picks:49 dd = FX / did; dd.mkdir(exist_ok=True)50 txt, meta = md_for(did, withhold_intro=True); (dd / "manuscript.md").write_text(txt)51 full, _ = md_for(did, withhold_intro=False); (dd / "manuscript_with_intro.md").write_text(full)52 d = segmentation.load_parsed(did); intro = next((s for s in d["sections"] if s.get("is_intro")), None)53 (dd / "published_introduction.txt").write_text("\n\n".join(p["text"] for p in segmentation.paragraphs(intro)) if intro else "")54 (dd / "meta.json").write_text(json.dumps(meta, indent=1)); manifest["finance_docs"].append(meta)55(FX / "manifest.json").write_text(json.dumps(manifest, indent=1)); print(json.dumps({k: v for k, v in manifest.items() if k != "finance_docs"}, indent=1)); print([(m["doc_id"], m["title"][:50], m["n_tables"]) for m in manifest["finance_docs"]])56