"""Generate appendix fragments: evaluator validation, environment records, ledgers, reproduction instructions, statistical details.""" import os, sys, csv, json, glob, re sys.path.insert(0, os.path.dirname(__file__)); from common import * APP = os.path.join(REPO, "paper", "appendices") from texutil import tex def evaluator_validation(): md = open(os.path.join(REPO, "reports", "evaluator_validation.md")).read() rows = [l for l in md.splitlines() if l.startswith("| F") or l.startswith("| T")] L = ["Every hidden oracle ships a seeded validation set (one or more independently written correct solutions whose replication script is re-run, and two to six wrong solutions). The table lists the outcome of \\texttt{evaluators/run\\_validations.py} executed before the freeze: PASS means every seeded case received the expected verdict (no false acceptance and no false rejection on the seeded set). The seeded sets bound gross evaluator errors; they do not cover the space of valid solutions.", "", r"\begin{longtable}{p{1.0in}p{0.6in}p{4.6in}}", r"\toprule", r"Task & Result & Last line of the validation output \\", r"\midrule", r"\endhead"] for r in rows: c = [x.strip() for x in r.strip("|").split("|")] L.append(f"{tex(c[0])} & {tex(c[1])} & {tex(c[2])[:220]} \\\\") L += [r"\bottomrule", r"\end{longtable}"] open(os.path.join(APP, "evaluator_validation.tex"), "w").write("\n".join(L) + "\n") def environment(): envs = {} for p in glob.glob(os.path.join(REPO, "runs", "*", "R*", "prepare.json")): try: e = json.load(open(p))["node_environment"]; envs[e["node"]] = e except Exception: pass L = ["Controller host: MacBook (Apple silicon, arm64), macOS 27.0, Claude Code 2.1.314 (npm), Python 3.12 virtual environment with numpy 2.3.4, pandas 2.3.3, scipy 1.16.2, statsmodels 0.14.5, matplotlib 3.10.7. Executor nodes (all `claude --bare`, model requested \\texttt{claude-opus-5-5}, model reported by the client in every session's \\texttt{modelUsage}: \\texttt{claude-opus-5-5}):", "", r"\begin{tabular}{llllll}", r"\toprule", r"Node & Host & macOS & Client & Python / numpy / pandas / scipy / statsmodels & Gate SHA-256 (prefix) \\", r"\midrule"] for n, e in sorted(envs.items()): L.append(f"{n} & {tex(e.get('host',''))} & {tex(e.get('os',''))} & {tex(e.get('claude',''))} & {tex(e.get('py',''))} & {tex(e.get('gate',''))} \\\\") L += [r"\bottomrule", r"\end{tabular}", "", "Official documentation pages (memory, headless, cli-reference, hooks, permissions) were retrieved on 2026-10-29 and are archived in \\texttt{docs/claude-code-docs/}. The client exposes no temperature or random-seed control; costs are the client's list-price estimates (\\texttt{total\\_cost\\_usd}), which Anthropic documents as estimates that can differ from billing."] open(os.path.join(APP, "environment.tex"), "w").write("\n".join(L) + "\n") def ledgers(): L = [r"Timestamps in both ledgers were entered by the builder to the minute; the git commits referenced in the amendment rows are the authoritative record of when each change was made.", "", r"\subsection*{Human interventions}", r"\begin{longtable}{p{0.5in}p{1.1in}p{0.9in}p{0.8in}p{2.1in}p{0.9in}}", r"\toprule", r"ID & UTC & Actor & Category & Description & Duration / basis \\", r"\midrule", r"\endhead"] for r in csv.DictReader(open(os.path.join(REPO, "reports", "interventions.csv"))): L.append(f"{tex(r['intervention_id'])} & {tex(r['timestamp_utc'][:16])} & {tex(r['actor'])} & {tex(r['category'])} & {tex(r['description'])[:260]} & {tex(r['duration_minutes'])} / {tex(r['measurement_basis'])[:60]} \\\\") L += [r"\bottomrule", r"\end{longtable}", r"\subsection*{Amendments to frozen designs}", r"\begin{longtable}{p{0.5in}p{0.9in}p{0.6in}p{0.9in}p{3.4in}}", r"\toprule", r"ID & UTC & Phase & Timing & Description and rationale \\", r"\midrule", r"\endhead"] for r in csv.DictReader(open(os.path.join(REPO, "protocol", "amendments.csv"))): L.append(f"{tex(r['amendment_id'])} & {tex(r['timestamp_utc'][:16])} & {tex(r['phase'])} & {tex(r['before_or_after_results']).replace('_', ' ')} & {tex(r['description'])[:420]} \\textit{{Rationale:}} {tex(r['rationale'])[:260]} \\\\") L += [r"\bottomrule", r"\end{longtable}"] open(os.path.join(APP, "ledgers.tex"), "w").write("\n".join(L) + "\n") def repro(): L = [r"Three modes are distinguished (Section~\ref{sec:limitations}). All commands are run from the repository root with the Python virtual environment in \texttt{.venv}; every command below was executed during this study.", "", r"\paragraph{Infrastructure tests (no provider access).} \texttt{make test} runs the state-machine, event, gate, evaluator and bundle-hygiene tests. \texttt{.venv/bin/python evaluators/run\_validations.py} re-runs every oracle's seeded validation set (requires the hidden oracles under \texttt{\textasciitilde/wp29-oracles/}).", r"\paragraph{Artefact replay (no provider access).} \texttt{make analyze} rebuilds the analysis dataset from \texttt{runs/state.sqlite}, recomputes every estimate, table and figure, and \texttt{make paper} recompiles the manuscript; \texttt{release/validate\_release.py} verifies the hashes of the released files and re-runs the headline analysis in a clean copy.", r"\paragraph{Fresh live replication (requires an Anthropic API key and executor hosts).} Install the gate and key file on the nodes (\texttt{make deploy}), probe isolation (\texttt{make probe}), then register and run an assignment: \texttt{python -m pgar.cli assign --split confirmatory --tasks --repeats 3 --seed 301} and \texttt{python -m pgar.cli run --split confirmatory --workers 8}. Model stochasticity is not seedable; results of a fresh replication are expected to differ run by run and should be compared at the task-cluster level with the intervals reported here."] open(os.path.join(APP, "repro.tex"), "w").write("\n".join(L) + "\n") def stats(): L = [r"\subsection*{Failed criteria and violations by task and arm}", r"\IfFileExists{../results/tables/tab_criteria_confirmatory.tex}{\input{../results/tables/tab_criteria_confirmatory}}{Not yet generated.}", "", r"\subsection*{Claim-level endpoint}", "All claims flagged as unsupported by the automatic rule belong to the F6-conf-04 and F3-conf-02 oracle defects; the apparent gradient across arms is therefore evaluator noise.", "", r"\IfFileExists{../results/tables/tab_claims_confirmatory.tex}{\input{../results/tables/tab_claims_confirmatory}}{}", "", r"\subsection*{Robustness of the primary contrast}", r"\IfFileExists{../results/tables/tab_robustness.tex}{\input{../results/tables/tab_robustness}}{Not yet generated.}", "", r"\subsection*{Order and node effects}", r"\IfFileExists{../results/tables/tab_order_node.tex}{\input{../results/tables/tab_order_node}}{Not yet generated.}"] open(os.path.join(APP, "stats.tex"), "w").write("\n".join(L) + "\n") if __name__ == "__main__": evaluator_validation(); environment(); ledgers(); repro(); stats(); print("appendix fragments written")