1{2 "critical_violations": [],3 "evaluated_utc": "2026-10-30T05:36:03.074889+00:00",4 "evaluator_version": "T1-forecast/1.0",5 "notes": "Deterministic evaluation; reproducibility filled by the harness. Real-time discipline is verified row by row on the baseline forecasts (origin vintage vs current vintage) and by chronology of forecasts.csv.",6 "oracle_sha256": "d4af22cc34534fe1fafe7b55d3f397c809c89bca2c564c41196be99c54d21740",7 "quality_components": {8 "baseline": 1.0,9 "correctness": 1.0,10 "integrity": 1.0,11 "reporting": 1.0,12 "reproducibility": 1.013 },14 "quality_score": 1.0,15 "receipt_sha256": "45bc1358f244eb68e3c511f9427d48ab507334d7d2fbb80863336bc37247cb97",16 "required_criteria": [17 {18 "description": "naive RMSE/MAE at h=1,3 equal the leak-free real-time reference (abs 2e-3)",19 "expected": {20 "mae_naive_h1": 0.8487221880004494,21 "mae_naive_h3": 1.2004486337876656,22 "rmse_naive_h1": 0.9968743518607753,23 "rmse_naive_h3": 1.420040373752046824 },25 "id": "naive_exact",26 "observed": {27 "mae_naive_h1": 0.8487221999999999,28 "mae_naive_h3": 1.2004486,29 "rmse_naive_h1": 0.996874361153601,30 "rmse_naive_h3": 1.420040301975123331 },32 "passed": true,33 "tolerance": {34 "mae_naive_h1": {35 "abs": 0.00236 },37 "mae_naive_h3": {38 "abs": 0.00239 },40 "rmse_naive_h1": {41 "abs": 0.00242 },43 "rmse_naive_h3": {44 "abs": 0.00245 }46 }47 },48 {49 "description": "seasonal-naive RMSE/MAE at h=1,3 equal the reference (abs 2e-3)",50 "expected": {51 "mae_snaive_h1": 1.3066698762542999,52 "mae_snaive_h3": 1.065868691479163,53 "rmse_snaive_h1": 1.6745428991342528,54 "rmse_snaive_h3": 1.316078898955204655 },56 "id": "snaive_exact",57 "observed": {58 "mae_snaive_h1": 1.3066698,59 "mae_snaive_h3": 1.0658687,60 "rmse_snaive_h1": 1.674542814799132,61 "rmse_snaive_h3": 1.316078860822519362 },63 "passed": true,64 "tolerance": {65 "mae_snaive_h1": {66 "abs": 0.00267 },68 "mae_snaive_h3": {69 "abs": 0.00270 },71 "rmse_snaive_h1": {72 "abs": 0.00273 },74 "rmse_snaive_h3": {75 "abs": 0.00276 }77 }78 },79 {80 "description": "AR(1) RMSE/MAE at h=1,3 within 0.03 of the real-time reference (sample-convention slack; rejects the final-vintage implementation)",81 "expected": {82 "mae_ar1_h1": 0.7976510573253133,83 "mae_ar1_h3": 0.9526895961443085,84 "rmse_ar1_h1": 0.9405244111185092,85 "rmse_ar1_h3": 1.18711781474230186 },87 "id": "ar1_close",88 "observed": {89 "mae_ar1_h1": 0.7976509999999999,90 "mae_ar1_h3": 0.9526893999999999,91 "rmse_ar1_h1": 0.940524354787796,92 "rmse_ar1_h3": 1.187117637092634293 },94 "passed": true,95 "tolerance": {96 "mae_ar1_h1": {97 "abs": 0.0398 },99 "mae_ar1_h3": {100 "abs": 0.03101 },102 "rmse_ar1_h1": {103 "abs": 0.03104 },105 "rmse_ar1_h3": {106 "abs": 0.03107 }108 }109 },110 {111 "description": "mean absolute revision of the h=1 targets (abs 2e-3)",112 "expected": {113 "mean_abs_revision_h1_targets": 0.5482198028388884114 },115 "id": "revision",116 "observed": {117 "mean_abs_revision_h1_targets": 0.5482198028388884118 },119 "passed": true,120 "tolerance": {121 "mean_abs_revision_h1_targets": {122 "abs": 0.002123 }124 }125 },126 {127 "description": "forecasts.csv: 80 rows, 20 per model, vintage_used = origin, target = last vintage month + h, actuals = first release, naive/snaive rows exact (2e-3), ar1 rows within 0.05",128 "expected": {129 "n_rows": 80,130 "per_model": 20131 },132 "id": "forecasts_chronology",133 "observed": {134 "actual_mismatches": 0,135 "ar1_row_mismatches": 0,136 "bad_rows": 0,137 "chronology_violations": 0,138 "counts": {139 "ar1": 20,140 "model": 20,141 "naive": 20,142 "snaive": 20143 },144 "n_rows": 80145 },146 "passed": true,147 "tolerance": null148 },149 {150 "description": "model RMSE/MAE and rel_improvement agree with forecasts.csv",151 "expected": null,152 "id": "model_errors_consistent",153 "observed": {154 "h1": {155 "mae_from_csv": 0.48633190000000004,156 "mae_reported": 0.48633190000000004,157 "rel_improvement_expected": 0.4479769362367869,158 "rel_improvement_reported": 0.4479769362367869,159 "rmse_from_csv": 0.5502976390310066,160 "rmse_reported": 0.5502976390310066161 },162 "h3": {163 "mae_from_csv": 0.9339774000000001,164 "mae_reported": 0.9339774000000001,165 "rel_improvement_expected": 0.19564278617186803,166 "rel_improvement_reported": 0.19564278617186803,167 "rmse_from_csv": 1.1422196608203694,168 "rmse_reported": 1.1422196608203694169 }170 },171 "passed": true,172 "tolerance": {173 "abs": 0.0001,174 "rel": 0.0001175 }176 },177 {178 "description": "Diebold-Mariano statistic/p-value (model vs naive, Bartlett lag h-1, normal p) agree with recomputation from forecasts.csv",179 "expected": null,180 "id": "dm_consistent",181 "observed": {182 "h1": {183 "p_recomputed": 0.0151414555400921,184 "p_reported": 0.015141455540092164,185 "stat_recomputed": 2.4289778933188826,186 "stat_reported": 2.4289778933188826187 },188 "h3": {189 "p_recomputed": 0.12196520179653339,190 "p_reported": 0.12196520179653336,191 "stat_recomputed": 1.5465773215484677,192 "stat_reported": 1.546577321548468193 }194 },195 "passed": true,196 "tolerance": {197 "p": {198 "abs": 0.03199 },200 "stat": {201 "abs": 0.12,202 "rel": 0.08,203 "rule": "either"204 }205 }206 },207 {208 "description": "conclusion codes agree with rel_improvement sign and dm p-value",209 "expected": null,210 "id": "conclusion_consistent",211 "observed": {212 "h1": {213 "code": "model_better_significant",214 "expected": "model_better_significant"215 },216 "h3": {217 "code": "model_better_not_significant",218 "expected": "model_better_not_significant"219 }220 },221 "passed": true,222 "tolerance": null223 },224 {225 "description": "table1.csv has 8 rows consistent with results.json; figure1.png exists",226 "expected": null,227 "id": "artifacts",228 "observed": {229 "figure": true,230 "table1_consistent": true,231 "table1_rows": 8232 },233 "passed": true,234 "tolerance": null235 }236 ],237 "run_id": "RT0011",238 "schema_error": "",239 "schema_valid": true,240 "scientific_success": true,241 "task_id": "T1-forecast"242}