1# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca2"""LaTeX fragment writers (booktabs, siunitx `\\num{}` cells). Each writer produces `<stem>.tex` and `<stem>.csv`.34Conventions5* A fragment is a complete `tabular` environment without the `table` float or caption (captions live in the paper).6* The last line of the file is `\\end{tabular}` — never a row ending in `\\\\` — so `\\input{}` can be followed by7 a `\\caption` or `\\label` without a spurious blank row.8* Numbers go through `\\num{...}` (siunitx); NaN / suppressed cells are printed as `--` inside braces `{--}`.9* Text cells are escaped (`_ % & # $`), arm names are prettified (`G_debate_r2` → `G (debate r2)`).10"""11from __future__ import annotations1213import re14from collections.abc import Sequence15from pathlib import Path1617import numpy as np18import pandas as pd1920from .metrics import METRIC_LABELS, PRIMARY_METRIC21from .stratify import STRATUM_LABELS2223MAIN_METRICS = [PRIMARY_METRIC, "mae_cad", "mdape", "rmse_log", "p90_abs_log_error", "p95_abs_log_error", "bias_log", "severe_rate", "n"]24COVERAGE_METRICS = ["coverage_50", "coverage_80", "coverage_90", "coverage_95", "mean_log_width_90", "pinball_log", "crps_log", "n_with_interval"]25COST_METRICS = ["cost_usd_mean", "latency_s_mean", "tokens_mean", "cost_usd_total", PRIMARY_METRIC, "n"]26DIGITS = {"n": 0, "n_with_interval": 0, "mae_cad": 0, "tokens_mean": 0, "cost_usd_total": 2, "cost_usd_mean": 4, "latency_s_mean": 1,27 "p_boot": 3, "p_holm": 3, "dm_p": 3, "dm_stat": 2}28_ESC = {"&": r"\&", "%": r"\%", "$": r"\$", "#": r"\#", "_": r"\_", "{": r"\{", "}": r"\}", "~": r"\textasciitilde{}", "^": r"\textasciicircum{}"}293031def escape(text: object) -> str:32 s = str(text)33 return "".join(_ESC.get(ch, ch) for ch in s)343536def arm_pretty(arm: str) -> str:37 """`G_debate_r2` → `G (debate r2)`; `A_hedonic` → `A (hedonic)`; other strings are escaped as-is."""38 m = re.match(r"^([A-Z])_(.+)$", str(arm))39 if not m:40 return escape(arm)41 rest = m.group(2).split("_")42 return f"{m.group(1)} ({escape(' '.join(rest))})"434445def num(x: object, digits: int = 3) -> str:46 try:47 v = float(x) # type: ignore[arg-type]48 except (TypeError, ValueError):49 return "{--}"50 if not np.isfinite(v):51 return "{--}"52 return rf"\num{{{v:.{digits}f}}}"535455def tabular(header: Sequence[str], rows: Sequence[Sequence[str]], colspec: str, *, midrules_after: Sequence[int] = ()) -> str:56 """Assemble a booktabs tabular; the returned string ends with `\\end{tabular}` (no trailing newline, no `\\\\`)."""57 lines = [rf"\begin{{tabular}}{{{colspec}}}", r"\toprule", " & ".join(header) + r" \\", r"\midrule"]58 for i, r in enumerate(rows):59 lines.append(" & ".join(r) + r" \\")60 if i in midrules_after and i < len(rows) - 1:61 lines.append(r"\midrule")62 lines += [r"\bottomrule", r"\end{tabular}"]63 return "\n".join(lines)646566def write_fragment(tex: str, path: Path, csv_df: pd.DataFrame | None = None) -> Path:67 path = Path(path)68 path.parent.mkdir(parents=True, exist_ok=True)69 body = tex.rstrip()70 assert not body.endswith(r"\\"), "fragment must not end with \\\\"71 path.write_text("% Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca\n% generated by scripts/20_evaluate.py — do not edit\n" + body + "\n", encoding="utf-8")72 if csv_df is not None:73 csv_df.to_csv(path.with_suffix(".csv"), index=False)74 return path757677def _label_col(df: pd.DataFrame) -> pd.Series:78 if "label" in df.columns:79 return df["label"].map(escape)80 return df["arm"].map(arm_pretty)818283def _metrics_table(df: pd.DataFrame, metrics: Sequence[str], path: Path, *, family_rule: bool = True) -> str:84 metrics = [m for m in metrics if m in df.columns]85 header = ["Arm"] + ["{" + METRIC_LABELS.get(m, escape(m)) + "}" for m in metrics]86 rows, mids = [], []87 labels = _label_col(df)88 fam = df["arm"].astype(str).str[0]89 for i, (_, r) in enumerate(df.iterrows()):90 rows.append([labels.iloc[i]] + [num(r[m], DIGITS.get(m, 3)) for m in metrics])91 if family_rule and i + 1 < len(df) and fam.iloc[i] <= "D" < fam.iloc[i + 1]:92 mids.append(i)93 tex = tabular(header, rows, "l" + "r" * len(metrics), midrules_after=mids)94 write_fragment(tex, path, df[["arm", "committee", "aggregation"] + metrics] if "committee" in df.columns else df[["arm"] + metrics])95 return tex969798def write_main_results(summary: pd.DataFrame, path: Path, metrics: Sequence[str] = MAIN_METRICS) -> str:99 """Arms × accuracy metrics (one row per arm; pass a pre-filtered summary with one row per arm)."""100 return _metrics_table(summary.sort_values("arm"), metrics, path)101102103def write_coverage(summary: pd.DataFrame, path: Path, metrics: Sequence[str] = COVERAGE_METRICS) -> str:104 """Arms × interval metrics; arms without intervals print `--`."""105 return _metrics_table(summary.sort_values("arm"), metrics, path)106107108def write_cost(summary: pd.DataFrame, path: Path, metrics: Sequence[str] = COST_METRICS) -> str:109 return _metrics_table(summary.sort_values("arm"), metrics, path)110111112def write_contrasts(contrasts: pd.DataFrame, path: Path) -> str:113 """Paired contrasts: Δ = metric(B) − metric(A) with bootstrap percentile CI, bootstrap p, Holm p and DM p."""114 header = ["Contrast (B vs A)", "{Metric}", "{$N$}", "{A}", "{B}", r"{$\Delta$}", "{95\\% CI}", "{$p$ (boot)}", "{$p$ (Holm)}", "{$p$ (DM)}"]115 rows = []116 for _, r in contrasts.iterrows():117 ci = rf"[{float(r['ci_lo']):.3f}, {float(r['ci_hi']):.3f}]" if np.isfinite(r["ci_lo"]) else "{--}"118 rows.append([f"{arm_pretty(r['arm_b'])} vs {arm_pretty(r['arm_a'])}", "{" + METRIC_LABELS.get(r["metric"], escape(r["metric"])) + "}",119 num(r["n"], 0), num(r["value_a"], DIGITS.get(r["metric"], 3)), num(r["value_b"], DIGITS.get(r["metric"], 3)),120 num(r["diff"], DIGITS.get(r["metric"], 3)), "{" + ci + "}", num(r["p_boot"], 3), num(r.get("p_holm", np.nan), 3), num(r.get("dm_p", np.nan), 3)])121 tex = tabular(header, rows, "llrrrrcrrr")122 write_fragment(tex, path, contrasts)123 return tex124125126def write_strata(strata: pd.DataFrame, path: Path, metric: str = PRIMARY_METRIC, arms: Sequence[str] | None = None) -> str:127 """Stratified table: rows = stratum × level (with n), columns = arms; suppressed cells print `--`."""128 s = strata[strata["metric"] == metric].copy()129 if "label" not in s.columns:130 s["label"] = s["arm"]131 if arms is not None:132 s = s[s["label"].isin(list(arms))]133 arm_list = list(arms) if arms is not None else sorted(s["label"].unique())134 if s.empty:135 tex = tabular(["Stratum", "{$n$}"], [[escape("no strata available"), "{--}"]], "lr")136 write_fragment(tex, path, s)137 return tex138 piv = s.pivot_table(index=["stratum", "level"], columns="label", values="value", aggfunc="first", sort=False)139 ns = s.groupby(["stratum", "level"], sort=False)["n"].max()140 header = ["Stratum", "{$n$}"] + ["{" + arm_pretty(a) + "}" for a in arm_list]141 rows, mids = [], []142 order = [st for st in STRATUM_LABELS if st in s["stratum"].unique()]143 i = -1144 for st in order:145 block = s[s["stratum"] == st].drop_duplicates(["level"])146 for _, lv in block.iterrows():147 i += 1148 key = (st, lv["level"])149 vals = [num(piv.loc[key, a] if a in piv.columns else np.nan, DIGITS.get(metric, 3)) for a in arm_list]150 rows.append([f"{escape(STRATUM_LABELS.get(st, st))}: {escape(lv['level'])}", num(ns.loc[key], 0)] + vals)151 mids.append(i)152 tex = tabular(header, rows, "lr" + "r" * len(arm_list), midrules_after=mids)153 write_fragment(tex, path, s)154 return tex155156157FAITH_COLS = [("cited_share_valid", "Valid ids"), ("share_turns_adj_ok", "Adj. $\\pm$40\\%"), ("point_in_hull", "In hull"),158 ("quantile_sane", "Quantiles ok"), ("summary_words", "Words"), ("iqr_log_round0", "IQR$_0$"),159 ("iqr_log_final", "IQR$_R$"), ("anchoring_index", "Anchoring"), ("share_toward_median", "To median")]160161162def write_faithfulness(faith: pd.DataFrame, path: Path) -> str:163 cols = [(c, l) for c, l in FAITH_COLS if c in faith.columns]164 header = ["Arm", "{Committee}", "{Turns}"] + ["{" + l + "}" for _, l in cols]165 rows = []166 for _, r in faith.iterrows():167 rows.append([arm_pretty(r["arm"]), escape(r.get("committee", "")), num(r.get("n_turns", np.nan), 0)]168 + [num(r[c], 0 if c == "summary_words" else 3) for c, _ in cols])169 if not rows:170 rows = [["{--}"] * len(header)]171 tex = tabular(header, rows, "ll" + "r" * (len(header) - 2))172 write_fragment(tex, path, faith)173 return tex174