1{2 "critical_violations": [],3 "evaluated_utc": "2026-10-30T04:35:49.855784+00:00",4 "evaluator_version": "F2-conf-01/1.0",5 "notes": "Deterministic evaluation; reproducibility component is filled by the harness.",6 "oracle_sha256": "d5628c5e913ac513d40d3de76007ffbfc44f8339cdecefe099263a0468e16d7a",7 "quality_components": {8 "baseline": 1.0,9 "correctness": 1.0,10 "integrity": 1.0,11 "reporting": 0.8333333333333334,12 "reproducibility": 1.013 },14 "quality_score": 0.975,15 "receipt_sha256": "15d17dfb913d0896fce2dcc7568381604cd866ec0d14df25e2beae31ced8cf84",16 "required_criteria": [17 {18 "description": "Student-weighted difference in means (OLS of score on treated) equals reference to 1e-4",19 "expected": {20 "tau_hat": 0.961117999999993521 },22 "id": "tau_point",23 "observed": {24 "tau_hat": 0.961117999999999925 },26 "passed": true,27 "tolerance": {28 "tau_hat": {29 "abs": 0.000130 }31 }32 },33 {34 "description": "Reported SE within +/-15% of the CR1 school-clustered reference (accepts CR0/CR1/CR2/CR3, school-mean t-test, random-intercept model, wild cluster bootstrap; rejects i.i.d./HC SEs at ~0.41x and over-inflated SEs)",35 "expected": {36 "se_cluster": 0.728957839251160237 },38 "id": "se_valid",39 "observed": {40 "se_cluster": 0.728957839251165841 },42 "passed": true,43 "tolerance": {44 "se_cluster": {45 "rel": 0.1546 }47 }48 },49 {50 "description": "Two-sided p-value for tau = 0 within 0.06 of the clustered t(G-2) reference",51 "expected": {52 "p_value": 0.1952341776804169353 },54 "id": "p_value",55 "observed": {56 "p_value": 0.1950331918035739257 },58 "passed": true,59 "tolerance": {60 "p_value": {61 "abs": 0.0662 }63 }64 },65 {66 "description": "95% CI bounds within 0.12 of the reference (critical-value and CR-convention differences)",67 "expected": {68 "tau_ci_high": 2.4368159955179216,69 "tau_ci_low": -0.514579995517934470 },71 "id": "ci",72 "observed": {73 "tau_ci_high": 2.435574402542949,74 "tau_ci_low": -0.513338402542949575 },76 "passed": true,77 "tolerance": {78 "tau_ci_high": {79 "abs": 0.1280 },81 "tau_ci_low": {82 "abs": 0.1283 }84 }85 },86 {87 "description": "Intraclass correlation of the residual within 0.06 of the ANOVA reference (ANOVA, ML and REML estimators all accepted)",88 "expected": {89 "icc_hat": 0.2349873934722339490 },91 "id": "icc",92 "observed": {93 "icc_hat": 0.2402970751227607494 },95 "passed": true,96 "tolerance": {97 "icc_hat": {98 "abs": 0.0699 }100 }101 },102 {103 "description": "n_schools, n_treated_schools, n_analysed exact",104 "expected": {105 "n_analysed": 1000,106 "n_schools": 40,107 "n_treated_schools": 20108 },109 "id": "counts",110 "observed": {111 "n_analysed": 1000,112 "n_schools": 40,113 "n_treated_schools": 20114 },115 "passed": true,116 "tolerance": {117 "n_analysed": {118 "abs": 0119 },120 "n_schools": {121 "abs": 0122 },123 "n_treated_schools": {124 "abs": 0125 }126 }127 },128 {129 "description": "effect_significant_5pct must be false (the valid clustered test does not reject at 5%)",130 "expected": false,131 "id": "not_significant",132 "observed": false,133 "passed": true,134 "tolerance": null135 }136 ],137 "run_id": "RC0110",138 "schema_error": "",139 "schema_valid": true,140 "scientific_success": true,141 "task_id": "F2-conf-01"142}