1"""Rubric-label sensitivity (amendment A-005, AFTER results, sensitivity only — the frozen primary scoring is untouched).23Two confirmatory tasks failed in (almost) every arm for reasons that lie in the oracle, not in the executors:4 * F1-conf-01: the source has no CPI index for 2025-10, so the 12-month growth of one window month is undefined; all 12 executors5 computed every required value correctly and reported feasibility = 'partial' with that reason; the frozen oracle demanded 'complete'.6 Corrected rule: 'partial' is accepted when n_months_missing_growth == 1 (or the reason names 2025-10) and every criterion passes.7 * F6-conf-04: the generator produced 12 firms with negative training hours; 11 of 12 executors detected and excluded them (documented in8 exclusions.csv/data_issues), so n_analysed = 1188 and the baseline slope differs from the all-rows reference. Corrected rule: n_analysed in9 {1188, 1200}; the baseline criterion accepts either the all-rows reference or an independently computed 1188-row reference (numpy OLS, HC1).10Re-scores every run of the two tasks under the corrected rules and recomputes the primary contrasts; outputs go to results/estimates/sensitivity_rubric_*.csv.11"""12import os, sys, json, glob13sys.path.insert(0, os.path.dirname(__file__)); from common import *14sys.path.insert(0, os.path.join(REPO, "src"))15from primary_analysis import arm_table, contrasts1617def ref_1188():18 d = pd.read_csv(os.path.join(REPO, "tasks", "public_bundles", "F6-conf-04", "data", "firms.csv")); d = d[d.training_hours >= 0]19 X = np.column_stack([np.ones(len(d)), d.training_hours.values]); y = d.productivity_growth.values20 XtX_inv = np.linalg.inv(X.T @ X); b = XtX_inv @ X.T @ y; e = y - X @ b21 meat = (X * (e[:, None] ** 2)).T @ X; V = XtX_inv @ meat @ XtX_inv * len(d) / (len(d) - 2)22 se = float(np.sqrt(V[1, 1])); return {"n": int(len(d)), "naive_estimate": float(b[1]), "naive_se_hc1": se, "naive_ci_low": float(b[1] - 1.96 * se), "naive_ci_high": float(b[1] + 1.96 * se)}2324def rescore(rec, res, task):25 rec = json.loads(json.dumps(rec)); changed = []26 if task == "F1-conf-01":27 if res.get("feasibility") == "partial" and (res.get("n_months_missing_growth") == 1 or "2025-10" in str(res.get("feasibility_reason"))):28 before = len(rec["critical_violations"]); rec["critical_violations"] = [v for v in rec["critical_violations"] if v["id"] != "false_complete"]29 if len(rec["critical_violations"]) != before: changed.append("false_complete waived (honest partial: 2025-10 undefined)")30 if task == "F6-conf-04":31 r2 = ref_1188()32 for c in rec["required_criteria"]:33 if c["id"] == "counts" and res.get("n_analysed") in (1188, 1200) and not c["passed"]:34 c["passed"] = True; changed.append("counts: 1188 accepted (impossible negative hours excluded)")35 if c["id"] == "naive" and not c["passed"]:36 ok = abs(res.get("naive_estimate", 1e9) - r2["naive_estimate"]) <= 0.003 and abs(res.get("naive_ci_low", 1e9) - r2["naive_ci_low"]) <= 0.006 + 0.004 and abs(res.get("naive_ci_high", 1e9) - r2["naive_ci_high"]) <= 0.006 + 0.00437 if ok: c["passed"] = True; changed.append("naive: 1188-row reference accepted")38 rec["scientific_success"] = all(c["passed"] for c in rec["required_criteria"]) and not rec["critical_violations"]39 rec["quality_components"]["correctness"] = sum(c["passed"] for c in rec["required_criteria"]) / max(1, len(rec["required_criteria"]))40 rec["quality_components"]["integrity"] = 0.0 if rec["critical_violations"] else 1.041 return rec, changed4243def main():44 df = load_runs("confirmatory"); rows = []45 for _, r in df.iterrows():46 rd = os.path.join(REPO, "runs", "confirmatory", r.run_id)47 try: rec = json.load(open(os.path.join(rd, "evaluation_receipt.json"))); res = json.load(open(os.path.join(rd, "out", "results.json")))48 except Exception: rows.append({"run_id": r.run_id, "success_frozen": r.scientific_success, "success_corrected": r.scientific_success, "changed": ""}); continue49 if r.task_id in ("F1-conf-01", "F6-conf-04"):50 rec2, ch = rescore(rec, res, r.task_id)51 rows.append({"run_id": r.run_id, "success_frozen": r.scientific_success, "success_corrected": int(rec2["scientific_success"]), "changed": "; ".join(ch), "n_violations_corrected": len(rec2["critical_violations"]), "q_correctness_corrected": rec2["quality_components"]["correctness"]})52 else:53 rows.append({"run_id": r.run_id, "success_frozen": r.scientific_success, "success_corrected": r.scientific_success, "changed": "", "n_violations_corrected": r.n_violations, "q_correctness_corrected": r.q_correctness})54 s = pd.DataFrame(rows); s.to_csv(os.path.join(EST, "sensitivity_rubric_runs.csv"), index=False)55 df2 = df.merge(s[["run_id", "success_corrected", "n_violations_corrected", "q_correctness_corrected"]], on="run_id")56 df2["Y"] = ((df2.success_corrected.fillna(0) == 1) & (~df2.terminal_reason.fillna("").isin(["wall_clock_limit"]))).astype(int)57 df2["Y_any"] = df2["success_corrected"].fillna(0).astype(int); df2["viol"] = (df2.n_violations_corrected.fillna(0) > 0).astype(int)58 at = arm_table(df2); at.to_csv(os.path.join(EST, "sensitivity_rubric_arm_table.csv"), index=False)59 ct = contrasts(df2, "Y"); ct.to_csv(os.path.join(EST, "sensitivity_rubric_contrasts.csv"), index=False)60 json.dump(ref_1188(), open(os.path.join(EST, "sensitivity_rubric_F6-conf-04_reference_1188.json"), "w"), indent=1)61 print("changed runs:", int((s.success_frozen != s.success_corrected).sum())); print(at[["arm", "Y_task_weighted", "violation_rate", "quality_mean"]].round(3).to_string()); print(ct[["contrast", "role", "estimate", "ci_low", "ci_high", "perm_p"]].round(4).to_string())62if __name__ == "__main__": main()63