Repository Public Research preview
1#!/usr/bin/env python32"""Build paper figures and tables from evaluation reports and training logs (never from hand-typed numbers).34Inputs: artifacts/metrics/<name>/report.json (scripts/evaluate.py), artifacts/runs/<run>/train_log.jsonl.5Outputs: paper/figures/*.pdf, paper/tables/*.tex, paper/numbers.tex (macros), paper/figures/FIGURE_PROVENANCE.md6"""7from __future__ import annotations89import argparse10import re11import json12import sys13from pathlib import Path1415import matplotlib1617matplotlib.use("Agg")18import matplotlib.pyplot as plt # noqa: E4021920ROOT = Path(__file__).resolve().parents[1]21sys.path.insert(0, str(ROOT / "src"))22from htr.evaluation.plotstyle import INK2, SEQUENTIAL_BLUE, apply, system_color # noqa: E40223from htr.evaluation.stylometry import FEATURES # noqa: E40224from htr.utils.io import read_jsonl # noqa: E4022526PRETTY = {"B0_identity": "B0 identity", "B1_generic_paraphrase": "B1 paraphrase", "chrf": "chrF++", "edit_sim": "edit similarity", "rougeL": "ROUGE-L", "exact_match": "exact match", "number_recall": "number recall", "placeholders_exact": "placeholders exact"}272829def pretty(s: str) -> str:30 return PRETTY.get(s, s.replace("htr-", "").replace("_", " "))313233SPLIT_LABEL = "evaluation split"343536def fig_overall(report: dict, out: Path, metrics=("chrf", "edit_sim", "number_recall")) -> None:37 systems = list(report["systems"])38 fig, axes = plt.subplots(1, len(metrics), figsize=(2.6 * len(metrics), 2.6), constrained_layout=True)39 for ax, m in zip(axes, metrics):40 for i, s in enumerate(systems):41 o = report["systems"][s]["overall"][m]42 ax.errorbar(o["mean"], i, xerr=[[o["mean"] - o["ci_low"]], [o["ci_high"] - o["mean"]]], fmt="o", color=system_color(s), ecolor=system_color(s), elinewidth=1.2, capsize=2)43 ax.text(o["mean"], i + 0.28, f"{o['mean']:.1f}" if m == "chrf" else f"{o['mean']:.3f}", ha="center", va="bottom", fontsize=7, color=INK2)44 ax.set_yticks(range(len(systems)))45 ax.set_yticklabels([pretty(s) for s in systems] if ax is axes[0] else [""] * len(systems))46 ax.set_xlabel(pretty(m))47 ax.set_ylim(-0.6, len(systems) - 0.2)48 ax.grid(axis="y", visible=False)49 fig.suptitle(f"Reconstruction quality on the {SPLIT_LABEL} (mean, 95 % document-clustered bootstrap CI)", fontsize=9)50 fig.savefig(out)51 plt.close(fig)525354def fig_by_group(report: dict, out: Path, group: str, metric: str = "chrf") -> None:55 systems = list(report["systems"])56 keys = sorted({k for s in systems for k in report["systems"][s][group]}, key=lambda k: -report["systems"][systems[0]][group].get(k, {"n": 0})["n"])57 fig, ax = plt.subplots(figsize=(max(3.2, 0.55 * len(keys) + 1.5), 2.6), constrained_layout=True)58 w = 0.8 / len(systems)59 for i, s in enumerate(systems):60 xs, ys, lo, hi = [], [], [], []61 for j, k in enumerate(keys):62 o = report["systems"][s][group].get(k)63 if not o:64 continue65 xs.append(j + (i - (len(systems) - 1) / 2) * w)66 ys.append(o[metric]["mean"])67 lo.append(o[metric]["mean"] - o[metric]["ci_low"])68 hi.append(o[metric]["ci_high"] - o[metric]["mean"])69 ax.errorbar(xs, ys, yerr=[lo, hi], fmt="o", color=system_color(s), elinewidth=1.0, capsize=1.5, label=pretty(s))70 ax.set_xticks(range(len(keys)))71 ax.set_xticklabels([f"{k}\n(n={report['systems'][systems[0]][group][k]['n']})" for k in keys], fontsize=7)72 ax.set_ylabel(pretty(metric))73 ax.grid(axis="x", visible=False)74 ax.legend(ncol=min(3, len(systems)), loc="upper center", bbox_to_anchor=(0.5, -0.28))75 fig.savefig(out)76 plt.close(fig)777879def fig_stylometry(report: dict, out: Path) -> None:80 sty = report.get("stylometry", {})81 means = sty.get("feature_means", {})82 if not means:83 return84 systems = [s for s in report["systems"] if s in sty]85 orig = means["original"]86 sds = sty.get("feature_sds", {}).get("original", {f: 1.0 for f in FEATURES})87 rows = ["rewrite", *systems]88 fig, ax = plt.subplots(figsize=(6.2, 3.6), constrained_layout=True)89 for i, r in enumerate(rows):90 m = means["rewrite"] if r == "rewrite" else sty[r]["feature_means"]91 # standardised mean shift (in SDs of the originals; floor avoids blow-ups on near-constant features)92 vals = [(m[f] - orig[f]) / max(sds.get(f, 1.0), 0.05 * (abs(orig[f]) + 1e-9), 1e-6) for f in FEATURES]93 from htr.evaluation.plotstyle import system_marker9495 ax.plot(range(len(FEATURES)), vals, marker=("x" if r == "rewrite" else system_marker(r, systems)), color=("#52514e" if r == "rewrite" else system_color(r)), label=("LLM rewrite" if r == "rewrite" else pretty(r)), linewidth=1.2)96 ax.axhline(0, color="#c3c2b7", linewidth=0.8)97 ax.set_xticks(range(len(FEATURES)))98 ax.set_xticklabels([f.replace("_", " ") for f in FEATURES], rotation=60, ha="right", fontsize=7)99 ax.set_ylabel("mean shift vs originals (SD units)")100 ax.grid(axis="x", visible=False)101 ax.legend(ncol=2, loc="best")102 fig.savefig(out)103 plt.close(fig)104105106def fig_training(runs: list[Path], out: Path) -> None:107 fig, ax = plt.subplots(figsize=(4.2, 2.6), constrained_layout=True)108 for i, run in enumerate(runs):109 log = list(read_jsonl(run / "train_log.jsonl"))110 ev = [(r["step"], r["eval"]["val_ce"]) for r in log if "eval" in r]111 tr = [(r["step"], r["ce"]) for r in log if "ce" in r]112 col = SEQUENTIAL_BLUE[min(len(SEQUENTIAL_BLUE) - 1, 2 + i)] if len(runs) > 3 else system_color(run.name)113 if tr:114 ax.plot(*zip(*tr), color=col, alpha=0.35, linewidth=0.8)115 if ev:116 ax.plot(*zip(*ev), marker="o", color=col, label=run.name.replace("pilot_", ""))117 ax.set_xlabel("optimizer step")118 ax.set_ylabel("token cross-entropy (val = markers, train = faint)")119 ax.legend()120 fig.savefig(out)121 plt.close(fig)122123124def table_overall(report: dict, out: Path, metrics=("chrf", "edit_sim", "rougeL", "number_recall", "placeholders_exact")) -> None:125 lines = ["\\begin{tabular}{l r " + " ".join("c" for _ in metrics) + "}", "\\toprule", "System & $n$ & " + " & ".join(pretty(m) for m in metrics) + " \\\\", "\\midrule"]126 for s, d in report["systems"].items():127 o = d["overall"]128 cells = [f"{o[m]['mean']:.2f} [{o[m]['ci_low']:.2f}, {o[m]['ci_high']:.2f}]" if m == "chrf" else f"{o[m]['mean']:.3f} [{o[m]['ci_low']:.3f}, {o[m]['ci_high']:.3f}]" for m in metrics]129 lines.append(f"{pretty(s)} & {o['n']} & " + " & ".join(cells) + " \\\\")130 lines += ["\\bottomrule", "\\end{tabular}"]131 out.write_text("\n".join(lines) + "\n")132133134_DIGIT_WORDS = {"0": "Zero", "1": "One", "2": "Two", "3": "Three", "4": "Four", "5": "Five", "6": "Six", "7": "Seven", "8": "Eight", "9": "Nine"}135136137def macro_key(s: str) -> str:138 """Collision-free LaTeX macro fragment: letters kept (title-cased per token), digits spelled out, everything else is a139 token boundary. ``htr-m1-tiny-s2`` → ``HtrMOneTinySTwo`` (dropping digits made m1/m2 and s2/s3 identical)."""140 out = []141 for tok in re.split(r"[^A-Za-z0-9]+", s):142 if not tok:143 continue144 out.append("".join(_DIGIT_WORDS[ch] if ch.isdigit() else ch for ch in tok[0].upper() + tok[1:]))145 return "".join(out)146147148def numbers_tex(report: dict, out: Path, tag: str) -> None:149 def macro(name: str, val: str) -> str:150 return f"\\newcommand{{\\{name}}}{{{val}}}"151 lines = [f"% generated by scripts/build_figures.py from {tag} — do not edit"]152 seen: set[str] = set()153 for s, d in report["systems"].items():154 key = macro_key(s)155 for m, v in d["overall"].items():156 if isinstance(v, dict):157 mk = macro_key(m)158 lines.append(macro(f"{tag}{key}{mk}", f"{v['mean']:.2f}" if m == "chrf" else f"{v['mean']:.3f}"))159 lines.append(macro(f"{tag}{key}{mk}Lo", f"{v['ci_low']:.2f}" if m == "chrf" else f"{v['ci_low']:.3f}"))160 lines.append(macro(f"{tag}{key}{mk}Hi", f"{v['ci_high']:.2f}" if m == "chrf" else f"{v['ci_high']:.3f}"))161 lines.append(macro(f"{tag}{key}N", str(d["overall"]["n"])))162 for s, dd in report.get("paired_vs_baseline", {}).items():163 key = macro_key(s)164 for m, v in dd.items():165 mk = macro_key(m)166 lines.append(macro(f"{tag}{key}{mk}Diff", f"{v['mean']:+.2f}" if m == "chrf" else f"{v['mean']:+.3f}"))167 lines.append(macro(f"{tag}{key}{mk}PHolm", f"{v.get('p_holm', float('nan')):.3f}"))168 sty = report.get("stylometry", {})169 if "rewrite_vs_original" in sty:170 lines.append(macro(f"{tag}WRewrite", f"{sty['rewrite_vs_original']['W1_mean']:.3f}"))171 for s in report["systems"]:172 if s in sty:173 key = macro_key(s)174 lines.append(macro(f"{tag}W{key}", f"{sty[s]['vs_original']['W1_mean']:.3f}"))175 names = [ln.split("{")[1].split("}")[0] for ln in lines if ln.startswith("\\newcommand")]176 dup = {n for n in names if names.count(n) > 1}177 if dup:178 raise SystemExit(f"duplicate macro names in {out}: {sorted(dup)[:5]}")179 out.write_text("\n".join(lines) + "\n")180181182def main() -> None:183 ap = argparse.ArgumentParser()184 ap.add_argument("--report", required=True, help="artifacts/metrics/<name>/report.json")185 ap.add_argument("--tag", default="Pilot", help="LaTeX macro prefix (letters only)")186 ap.add_argument("--runs", nargs="*", default=[], help="training run dirs for the learning-curve figure")187 ap.add_argument("--split-label", default="document-safe test split", help="e.g. 'validation split (exploratory)'")188 a = ap.parse_args()189 global SPLIT_LABEL190 SPLIT_LABEL = a.split_label191 apply()192 rep = json.loads(Path(a.report).read_text())193 from htr.evaluation.plotstyle import assign_system_colors194195 assign_system_colors(list(rep["systems"]))196 figs, tabs = ROOT / "paper/figures", ROOT / "paper/tables"197 figs.mkdir(exist_ok=True)198 tabs.mkdir(exist_ok=True)199 tag = a.tag200 fig_overall(rep, figs / f"fig_overall_{tag.lower()}.pdf")201 for g in ("by_variant", "by_role", "by_discipline", "by_era"):202 fig_by_group(rep, figs / f"fig_{g}_{tag.lower()}.pdf", g)203 fig_stylometry(rep, figs / f"fig_stylometry_{tag.lower()}.pdf")204 if a.runs:205 fig_training([Path(r) for r in a.runs], figs / f"fig_training_{tag.lower()}.pdf")206 table_overall(rep, tabs / f"tab_overall_{tag.lower()}.tex")207 numbers_tex(rep, ROOT / "paper" / f"numbers_{tag.lower()}.tex", tag)208 prov = figs / "FIGURE_PROVENANCE.md"209 prov.write_text((prov.read_text() if prov.exists() else "# Figure provenance\n\n") + f"- {tag}: generated from `{a.report}` (created {rep.get('created_at')}, git {rep.get('git', {}).get('commit')}) and runs {a.runs}\n")210 print("figures/tables/numbers written for", tag)211212213if __name__ == "__main__":214 main()215