1"""Generate reports/pilot_report.md from the state store (costs, durations, outcomes, gate/review statistics, scale decision)."""2import os, sys, json, datetime3sys.path.insert(0, os.path.dirname(__file__)); from common import *4def main():5 df = load_runs("pilot"); s = pd.read_csv(os.path.join(DATA, "sessions.csv")); s = s[s.run_id.isin(df.run_id)]6 g = pd.read_csv(os.path.join(DATA, "gate_receipts.csv")); g = g[g.run_id.isin(df.run_id)]; rv = pd.read_csv(os.path.join(DATA, "reviews.csv")); rv = rv[rv.run_id.isin(df.run_id)]7 L = ["# Pilot report (development tasks, 1 repeat per arm)", "", f"Generated {datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds')} from `results/analysis_data/*.csv` by `analysis/pilot_report.py`.", ""]8 L += [f"Runs assigned {len(df)}, evaluated {(df.status=='evaluated').sum()}, tasks {df.task_id.nunique()} ({', '.join(sorted(df.task_id.unique()))}). Model reported by the client: {', '.join(sorted(df.model_reported.dropna().unique()))}; client {', '.join(sorted(df.client_version.dropna().unique()))}.", ""]9 L += ["## Outcomes by arm", "", "| arm | runs | Y (task-weighted) | success (any) | runs with violation | reproducible | quality mean | mean cost USD | median cost | max cost | mean wall s | max wall s | sessions/run |", "|---|---|---|---|---|---|---|---|---|---|---|---|---|"]10 for a in ARMS:11 d = df[df.treatment == a]12 L.append(f"| {a} | {len(d)} | {task_weighted_rate(d,a):.2f} | {d.Y_any.mean():.2f} | {int(d.viol.sum())} | {d.reproducibility.mean():.2f} | {d.quality_score.mean():.3f} | {d.spent_usd.mean():.3f} | {d.spent_usd.median():.3f} | {d.spent_usd.max():.3f} | {d.spent_wall_s.mean():.0f} | {d.spent_wall_s.max():.0f} | {d.n_sessions.mean():.1f} |")13 L += ["", "## Outcomes by task", "", "| task | " + " | ".join(f"Y {a}" for a in ARMS) + " | " + " | ".join(f"cost {a}" for a in ARMS) + " |", "|---|" + "---|" * 8]14 for t in sorted(df.task_id.unique()):15 d = df[df.task_id == t].set_index("treatment")16 L.append(f"| {t} | " + " | ".join(str(int(d.loc[a, 'Y'])) if a in d.index else "–" for a in ARMS) + " | " + " | ".join(f"{d.loc[a, 'spent_usd']:.2f}" if a in d.index else "–" for a in ARMS) + " |")17 L += ["", "## Sessions", "", s.groupby(["role", "kind"]).agg(n=("session_id", "count"), cost_mean=("session_cost_usd", "mean"), turns_mean=("num_turns", "mean"), duration_s=("duration_ms", lambda x: x.mean()/1000)).round(2).to_markdown(), ""]18 L += ["## Gate receipts (controller-side)", "", g.groupby(["origin", "passed"]).size().rename("n").reset_index().to_markdown(index=False), "", "## Reviews (arm D)", "", rv.groupby("verdict").agg(n=("run_id", "count"), issues_mean=("n_issues", "mean")).to_markdown() if len(rv) else "none", ""]19 c = df.spent_usd.mean(); t = df.spent_wall_s.mean()20 for N, label in [(288, "24 tasks × 4 arms × 3 repeats"), (192, "24 × 4 × 2"), (96, "12 × 4 × 2")]:21 L.append(f"- Projected {label}: {N} runs ≈ {1.15*c*N:.0f} USD (1.15 × mean pilot cost {c:.3f}), ≈ {N*t/8/3600:.1f} h at 8 concurrent runs (mean pilot wall {t:.0f} s).")22 L += ["", "Scale decision rule (preregistration §6): choose the largest configuration whose projected cost fits the remaining envelope after a 150 USD reserve and whose projected time is ≤ 12 h."]23 open(os.path.join(REPO, "reports", "pilot_report.md"), "w").write("\n".join(L) + "\n"); print("\n".join(L[-5:]))24if __name__ == "__main__": main()25