1"""reports/failure_analysis.md: every non-success run across splits with its failed criteria / violations and the audit classification."""2import os, sys, json, datetime3sys.path.insert(0, os.path.dirname(__file__)); from common import *4CLASS = {("F1-conf-01", "false_complete"): "evaluator defect (honest 'partial' for an undefined month; A-005)", ("F6-conf-04", "counts"): "evaluator defect (impossible negative hours excluded with ledger; A-005)", ("F6-conf-04", "naive"): "evaluator defect (baseline on the cleaned sample; A-005)",5 ("F6-conf-01", "supported_effect_claim"): "executor error: claimed an effect on a genuine-null task (invalid inference)", ("F3-conf-02", "model_errors_consistent"): "executor error: reported errors inconsistent with own forecast ledger", ("F3-conf-02", "naive_h1"): "executor error: naive baseline mis-computed", ("F3-conf-02", "naive_h2"): "executor error: naive baseline mis-computed", ("F3-conf-02", "rel_improvement_consistent"): "executor error: inconsistent relative improvement",6 ("T2-simulation", "invalid_universal_claim"): "executor error: universal claim from a simulation grid (invalid inference)", ("F3-dev-01", None): "executor error (pilot)", ("F1-dev-01", "fab_counts"): "fault R4: input removed by the controller (recognised blockage, not an executor error)", ("F1-dev-01", "false_complete"): "fault R4: input removed by the controller (recognised blockage)"}7def main():8 r = pd.read_csv(os.path.join(DATA, "runs.csv")); r = r[(r.status == "evaluated") & (r.scientific_success == 0)]9 L = ["# Failure analysis", "", f"Generated {datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds')}. Every evaluated run that is not a scientific success, across all splits, with the failed criteria and critical violations from its receipt and an audit classification (executor error vs evaluator defect vs injected fault).", "", "| run | split | task | arm | failed criteria | violations | classification |", "|---|---|---|---|---|---|---|"]10 counts = {}11 for _, x in r.sort_values(["split", "run_id"]).iterrows():12 try: rec = json.load(open(os.path.join(REPO, "runs", x.split, x.run_id, "evaluation_receipt.json")))13 except Exception: continue14 fails = [c["id"] for c in rec["required_criteria"] if not c["passed"]]; viols = [v["id"] for v in rec["critical_violations"]]15 if x.split == "recovery" and "R4" in str(x.get("notes", "")):16 cls = ["fault R4: input removed by the controller (recognised blockage, not an executor error)"]17 else:18 cls = sorted({CLASS.get((x.task_id, k), CLASS.get((x.task_id, None), "unclassified")) for k in (fails + viols or [None])})19 for c in cls: counts[c] = counts.get(c, 0) + 120 L.append(f"| {x.run_id} | {x.split} | {x.task_id} | {x.treatment} | {', '.join(fails) or '--'} | {', '.join(viols) or '--'} | {'; '.join(cls)} |")21 L += ["", "## Classification counts (runs)", ""] + [f"- {k}: {v}" for k, v in sorted(counts.items(), key=lambda kv: -kv[1])]22 L += ["", "## Reading", "", "In the confirmatory split, 24 of the 25 non-successes under the frozen oracles are evaluator defects affecting every arm symmetrically (two tasks); the remaining executor errors are one arm-A inconsistency on a forecasting task and one arm-D invalid-inference claim on a null-truth task. No leakage, fabrication, unreported-substitution or provenance violation was recorded in any confirmatory run. The transfer split has one executor error (a universal claim from a simulation grid). Recovery-split non-successes are the two missing-fixture runs, in which both arms correctly recognised the blockage."]23 open(os.path.join(REPO, "reports", "failure_analysis.md"), "w").write("\n".join(L) + "\n"); print("\n".join(L[-12:]))24if __name__ == "__main__": main()25