1{2 "critical_violations": [],3 "evaluated_utc": "2026-10-30T05:38:26.946308+00:00",4 "evaluator_version": "F6-conf-04/1.0",5 "notes": "Deterministic evaluation (stdlib only); figure checked by PNG magic bytes, table by CSV parse; reproducibility is filled by the harness.",6 "oracle_sha256": "0c9223f343f191437ea26a918a2a77e21882e6c16a0a86b30fd200135eedc7b7",7 "quality_components": {8 "baseline": 0.5,9 "correctness": 0.75,10 "integrity": 1.0,11 "reporting": 1.0,12 "reproducibility": 1.013 },14 "quality_score": 0.85,15 "receipt_sha256": "31ddacaebd3a487e7982adae89b703733184f3d03562ad96371aed38bef4ffa4",16 "required_criteria": [17 {18 "description": "n_analysed = 1200",19 "expected": {20 "n_analysed": 120021 },22 "id": "counts",23 "observed": {24 "n_analysed": 118825 },26 "passed": false,27 "tolerance": {28 "n_analysed": {29 "abs": 030 }31 }32 },33 {34 "description": "unadjusted slope and its 95% CI match the reference (abs 0.003 / 0.006)",35 "expected": {36 "naive_ci_high": -0.10792450180373739,37 "naive_ci_low": -0.162454708281857,38 "naive_estimate": -0.135189605042797239 },40 "id": "naive",41 "observed": {42 "naive_ci_high": -0.10304397995314195,43 "naive_ci_low": -0.1603518326928757,44 "naive_estimate": -0.1316979063230088245 },46 "passed": false,47 "tolerance": {48 "naive_ci_high": {49 "abs": 0.00650 },51 "naive_ci_low": {52 "abs": 0.00653 },54 "naive_estimate": {55 "abs": 0.00356 }57 }58 },59 {60 "description": "confounder-adjusted slope within 0.003, SE within 10% (OLS/HC variants), CI bounds within 0.005",61 "expected": {62 "effect_ci_high": 0.27082150943837385,63 "effect_ci_low": 0.23362748302781414,64 "effect_estimate": 0.252224496233094,65 "effect_se": 0.00947885217664941766 },67 "id": "adjusted",68 "observed": {69 "effect_ci_high": 0.27276978871330976,70 "effect_ci_low": 0.2343066231767153,71 "effect_estimate": 0.25353820594501253,72 "effect_se": 0.00981221232634553673 },74 "passed": true,75 "tolerance": {76 "effect_ci_high": {77 "abs": 0.00578 },79 "effect_ci_low": {80 "abs": 0.00581 },82 "effect_estimate": {83 "abs": 0.00384 },85 "effect_se": {86 "rel": 0.187 }88 }89 },90 {91 "description": "coefficient on the skill index within 0.03",92 "expected": {93 "skill_index_coefficient": -4.63043640947126494 },95 "id": "skill_index_coef",96 "observed": {97 "skill_index_coefficient": -4.62462087934575698 },99 "passed": true,100 "tolerance": {101 "skill_index_coefficient": {102 "abs": 0.03103 }104 }105 },106 {107 "description": "naive (baseline) and adjusted (model) residual RMSE within 1-1.5%",108 "expected": {109 "baseline_metric": 3.860935650765891,110 "model_metric": 1.9773229272347403111 },112 "id": "fit_metrics",113 "observed": {114 "baseline_metric": 3.8601022336733584,115 "model_metric": 1.9758571828823268116 },117 "passed": true,118 "tolerance": {119 "baseline_metric": {120 "rel": 0.01121 },122 "model_metric": {123 "rel": 0.015124 }125 }126 },127 {128 "description": "conclusion code = positive_effect (the adjusted estimate)",129 "expected": "positive_effect",130 "id": "conclusion",131 "observed": "positive_effect",132 "passed": true,133 "tolerance": null134 },135 {136 "description": "out/figure1.png exists and is a valid PNG",137 "expected": "valid PNG",138 "id": "figure",139 "observed": true,140 "passed": true,141 "tolerance": null142 },143 {144 "description": "out/table1.csv parses with >= 2 data rows",145 "expected": ">= 2 data rows",146 "id": "table",147 "observed": 4,148 "passed": true,149 "tolerance": null150 }151 ],152 "run_id": "RA0022",153 "schema_error": "",154 "schema_valid": true,155 "scientific_success": false,156 "task_id": "F6-conf-04"157}