1{2 "critical_violations": [],3 "evaluated_utc": "2026-10-30T04:24:44.094947+00:00",4 "evaluator_version": "F3-conf-04/1.0",5 "notes": "Deterministic evaluation; reproducibility filled by the harness. HAC band recomputed from the executor's own h=4 loss differentials.",6 "oracle_sha256": "e4d1358d226ad618b91a80bf40bd5faca84e88da6cdfcca828d5c14cf5fc5e42",7 "quality_components": {8 "baseline": 1.0,9 "correctness": 1.0,10 "integrity": 1.0,11 "reporting": 1.0,12 "reproducibility": 1.013 },14 "quality_score": 1.0,15 "receipt_sha256": "3d63a11c126af4e294bdbd6080a238e0c1c4af74d4e7759a5434c96ad182e26f",16 "required_criteria": [17 {18 "description": "Seasonal-naive RMSE/MAE at h=1 exact (rel 1e-6)",19 "expected": {20 "mae_snaive_h1": 2.1374736750000003,21 "rmse_snaive_h1": 2.84912586015657222 },23 "id": "snaive_h1",24 "observed": {25 "mae_snaive_h1": 2.1374736750000003,26 "rmse_snaive_h1": 2.84912586015657227 },28 "passed": true,29 "tolerance": {30 "mae_snaive_h1": {31 "rel": 1e-0632 },33 "rmse_snaive_h1": {34 "rel": 1e-0635 }36 }37 },38 {39 "description": "Seasonal-naive RMSE/MAE at h=2 exact",40 "expected": {41 "mae_snaive_h2": 2.1259446750000004,42 "rmse_snaive_h2": 2.84667408037747243 },44 "id": "snaive_h2",45 "observed": {46 "mae_snaive_h2": 2.1259446750000004,47 "rmse_snaive_h2": 2.84667408037747248 },49 "passed": true,50 "tolerance": {51 "mae_snaive_h2": {52 "rel": 1e-0653 },54 "rmse_snaive_h2": {55 "rel": 1e-0656 }57 }58 },59 {60 "description": "Seasonal-naive RMSE/MAE at h=4 exact",61 "expected": {62 "mae_snaive_h4": 2.181266125,63 "rmse_snaive_h4": 2.882315939257531564 },65 "id": "snaive_h4",66 "observed": {67 "mae_snaive_h4": 2.181266125,68 "rmse_snaive_h4": 2.882315939257531569 },70 "passed": true,71 "tolerance": {72 "mae_snaive_h4": {73 "rel": 1e-0674 },75 "rmse_snaive_h4": {76 "rel": 1e-0677 }78 }79 },80 {81 "description": "n_origins = 40",82 "expected": {83 "n_origins": 4084 },85 "id": "counts",86 "observed": {87 "n_origins": 4088 },89 "passed": true,90 "tolerance": {91 "n_origins": {92 "abs": 093 }94 }95 },96 {97 "description": "forecasts.csv: 240 rows, targets = origin + h quarters, seasonal-naive rows exact, actuals = data",98 "expected": {99 "n_rows": 240100 },101 "id": "forecasts_chronology",102 "observed": {103 "actual_mismatches": 0,104 "bad_rows": 0,105 "counts": {106 "model_h1": 40,107 "model_h2": 40,108 "model_h4": 40,109 "snaive_h1": 40,110 "snaive_h2": 40,111 "snaive_h4": 40112 },113 "n_rows": 240114 },115 "passed": true,116 "tolerance": null117 },118 {119 "description": "model RMSE/MAE per horizon agree with forecasts.csv (rel 1e-5)",120 "expected": null,121 "id": "model_errors_consistent",122 "observed": {123 "h1": {124 "mae_from_csv": 1.5187712,125 "mae_reported": 1.5187711988356731,126 "rmse_from_csv": 2.191813404902365,127 "rmse_reported": 2.1918133412897713128 },129 "h2": {130 "mae_from_csv": 1.5503333,131 "mae_reported": 1.550333358372702,132 "rmse_from_csv": 2.460222520387272,133 "rmse_reported": 2.46022256417967134 },135 "h4": {136 "mae_from_csv": 1.5936096499999999,137 "mae_reported": 1.5936096377164786,138 "rmse_from_csv": 2.3396849355013485,139 "rmse_reported": 2.3396848872653107140 }141 },142 "passed": true,143 "tolerance": {144 "rel": 1e-05145 }146 },147 {148 "description": "rel_improvement = 1 - rmse_model/rmse_snaive (abs 1e-6)",149 "expected": null,150 "id": "rel_improvement_consistent",151 "observed": {152 "h1": {153 "expected": 0.23070673291725996,154 "reported": 0.23070673291725996155 },156 "h2": {157 "expected": 0.13575544838858333,158 "reported": 0.13575544838858333159 },160 "h4": {161 "expected": 0.18826216952885444,162 "reported": 0.18826216952885444163 }164 },165 "passed": true,166 "tolerance": {167 "abs": 1e-06168 }169 },170 {171 "description": "loss_diff_mean_h4 = mean(e_snaive^2 - e_model^2) at h=4 from forecasts.csv (rel 1e-5)",172 "expected": null,173 "id": "loss_diff_mean_h4",174 "observed": {175 "from_csv": 2.8336195762860754,176 "reported": 2.8336198020003365177 },178 "passed": true,179 "tolerance": {180 "rel": 1e-05181 }182 },183 {184 "description": "loss_diff_se_h4 equals (rel 3%) the HAC standard error recomputed from the executor's own h=4 loss differentials with the reported kernel and truncation lag (lag in [3, 8]; Bartlett with either weight convention, rectangular or Parzen; small-sample factor n/(n-1) covered)",185 "expected": {186 "informational_band_bartlett_3_8": [187 1.19660077915704,188 2.082285048244578189 ]190 },191 "id": "hac_se_h4",192 "observed": {193 "iid_se": 1.9967878428857722,194 "kernel": "bartlett",195 "lags": 3,196 "reported": 2.0822850037684537,197 "valid_ses_for_reported_kernel_and_lag": [198 2.082285048244578,199 2.1960009576531787200 ]201 },202 "passed": true,203 "tolerance": 0.03204 },205 {206 "description": "hac_lags_h4 integer in [3, 8]",207 "expected": [208 3,209 8210 ],211 "id": "hac_lags_h4",212 "observed": 3,213 "passed": true,214 "tolerance": null215 },216 {217 "description": "dm_stat_h4 = mean/se (rel 1e-3) and dm_pvalue_h4 consistent (normal or t(39), with/without HLN correction, abs 0.005)",218 "expected": {219 "pvalue_band": [220 0.1685698629143063,221 0.22678745801152528222 ]223 },224 "id": "dm_stat_h4",225 "observed": {226 "dm_pvalue": 0.17356986291430632,227 "dm_stat": 1.360822268264018228 },229 "passed": true,230 "tolerance": null231 }232 ],233 "run_id": "RC0075",234 "schema_error": "",235 "schema_valid": true,236 "scientific_success": true,237 "task_id": "F3-conf-04"238}