"""reports/failure_analysis.md: every non-success run across splits with its failed criteria / violations and the audit classification."""
import os, sys, json, datetime
sys.path.insert(0, os.path.dirname(__file__)); from common import *
CLASS = {("F1-conf-01", "false_complete"): "evaluator defect (honest 'partial' for an undefined month; A-005)", ("F6-conf-04", "counts"): "evaluator defect (impossible negative hours excluded with ledger; A-005)", ("F6-conf-04", "naive"): "evaluator defect (baseline on the cleaned sample; A-005)",
         ("F6-conf-01", "supported_effect_claim"): "evaluator defect (regex false positive on the conclusion code no_detectable_effect; A-007)", ("F3-conf-02", "model_errors_consistent"): "evaluator defect (metrics rounded to 4 decimals vs 1e-6 tolerance; A-007)", ("F3-conf-02", "naive_h1"): "evaluator defect (rounding vs 1e-6 tolerance; A-007)", ("F3-conf-02", "naive_h2"): "evaluator defect (rounding vs 1e-6 tolerance; A-007)", ("F3-conf-02", "rel_improvement_consistent"): "evaluator defect (rounding; A-007)",
         ("T2-simulation", "invalid_universal_claim"): "executor error: universal claim from a simulation grid (invalid inference)", ("F3-dev-01", None): "executor error (pilot)", ("F1-dev-01", "fab_counts"): "fault R4: input removed by the controller (recognised blockage, not an executor error)", ("F1-dev-01", "false_complete"): "fault R4: input removed by the controller (recognised blockage)"}
def main():
    r = pd.read_csv(os.path.join(DATA, "runs.csv")); r = r[(r.status == "evaluated") & (r.scientific_success == 0)]
    L = ["# Failure analysis", "", f"Generated {datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds')}. Every evaluated run that is not a scientific success, across all splits, with the failed criteria and critical violations from its receipt and an audit classification (executor error vs evaluator defect vs injected fault).", "", "| run | split | task | arm | failed criteria | violations | classification |", "|---|---|---|---|---|---|---|"]
    counts = {}
    for _, x in r.sort_values(["split", "run_id"]).iterrows():
        try: rec = json.load(open(os.path.join(REPO, "runs", x.split, x.run_id, "evaluation_receipt.json")))
        except Exception: continue
        fails = [c["id"] for c in rec["required_criteria"] if not c["passed"]]; viols = [v["id"] for v in rec["critical_violations"]]
        if x.split == "recovery" and "R4" in str(x.get("notes", "")):
            cls = ["fault R4: input removed by the controller (recognised blockage, not an executor error)"]
        else:
            cls = sorted({CLASS.get((x.task_id, k), CLASS.get((x.task_id, None), "unclassified")) for k in (fails + viols or [None])})
        for c in cls: counts[c] = counts.get(c, 0) + 1
        L.append(f"| {x.run_id} | {x.split} | {x.task_id} | {x.treatment} | {', '.join(fails) or '--'} | {', '.join(viols) or '--'} | {'; '.join(cls)} |")
    L += ["", "## Classification counts (runs)", ""] + [f"- {k}: {v}" for k, v in sorted(counts.items(), key=lambda kv: -kv[1])]
    L += ["", "## Reading", "", "In the confirmatory split, all 25 non-successes under the frozen oracles are evaluator defects (four tasks: an undefined feasibility label, a generator artefact, a rounding tolerance, a regex false positive; amendments A-005/A-007); under the corrected rubric every arm completes every task. No leakage, fabrication, unreported-substitution or provenance violation was recorded in any confirmatory run. The transfer split has one executor error (a universal claim from a simulation grid). Recovery-split non-successes are the two missing-fixture runs, in which both arms correctly recognised the blockage."]
    open(os.path.join(REPO, "reports", "failure_analysis.md"), "w").write("\n".join(L) + "\n"); print("\n".join(L[-12:]))
if __name__ == "__main__": main()