1{2 "critical_violations": [],3 "evaluated_utc": "2026-10-30T04:01:38.447226+00:00",4 "evaluator_version": "F3-dev-01/1.0",5 "notes": "Deterministic evaluation; reproducibility component is filled by the harness. Full-sample re-estimation leaks are NOT detectable on this series (documented in REFERENCE.md); the bound catches smoothing/foresight leaks.",6 "oracle_sha256": "3c7498c1f70346124c8233d46ab30a2eef957ec2e3437b34674287f93b43016a",7 "quality_components": {8 "baseline": 1.0,9 "correctness": 1.0,10 "integrity": 1.0,11 "reporting": 1.0,12 "reproducibility": 1.013 },14 "quality_score": 1.0,15 "receipt_sha256": "316c1851986d82a485e328de56757793af04cdcf1cbf580282b8e10864edcab1",16 "required_criteria": [17 {18 "description": "Naive RMSE and MAE at h=1 equal the exact reference (rel 1e-6); 35 evaluated pairs (target 2025-10 missing)",19 "expected": {20 "mae_naive_h1": 0.09142857142857148,21 "rmse_naive_h1": 0.1171080087538240322 },23 "id": "naive_h1",24 "observed": {25 "mae_naive_h1": 0.09142857142857148,26 "rmse_naive_h1": 0.1171080087538240327 },28 "passed": true,29 "tolerance": {30 "mae_naive_h1": {31 "rel": 1e-0632 },33 "rmse_naive_h1": {34 "rel": 1e-0635 }36 }37 },38 {39 "description": "Naive RMSE and MAE at h=3 equal the exact reference (rel 1e-6)",40 "expected": {41 "mae_naive_h3": 0.12000000000000005,42 "rmse_naive_h3": 0.1585649934344184343 },44 "id": "naive_h3",45 "observed": {46 "mae_naive_h3": 0.12000000000000005,47 "rmse_naive_h3": 0.1585649934344184348 },49 "passed": true,50 "tolerance": {51 "mae_naive_h3": {52 "rel": 1e-0653 },54 "rmse_naive_h3": {55 "rel": 1e-0656 }57 }58 },59 {60 "description": "n_origins = 36, n_evaluated_h1 = 35, n_evaluated_h3 = 35",61 "expected": {62 "n_evaluated_h1": 35,63 "n_evaluated_h3": 35,64 "n_origins": 3665 },66 "id": "counts",67 "observed": {68 "n_evaluated_h1": 35,69 "n_evaluated_h3": 35,70 "n_origins": 3671 },72 "passed": true,73 "tolerance": {74 "n_evaluated_h1": {75 "abs": 076 },77 "n_evaluated_h3": {78 "abs": 079 },80 "n_origins": {81 "abs": 082 }83 }84 },85 {86 "description": "forecasts.csv: 144 rows, target = origin + h, naive forecast = last observation <= origin, actual matches data",87 "expected": {88 "n_rows": 14489 },90 "id": "forecasts_chronology",91 "observed": {92 "actual_mismatches": 0,93 "model_rows": 72,94 "n_rows": 144,95 "naive_rows": 7296 },97 "passed": true,98 "tolerance": null99 },100 {101 "description": "model RMSE/MAE in results.json agree with forecasts.csv (rel 1e-5)",102 "expected": "recomputed from forecasts.csv",103 "id": "model_errors_consistent",104 "observed": {105 "h1": {106 "mae_from_csv": 0.09561928226000001,107 "mae_reported": 0.09561928226000001,108 "n": 35,109 "rmse_from_csv": 0.1197904141277979,110 "rmse_reported": 0.1197904141277979111 },112 "h3": {113 "mae_from_csv": 0.12090394601714284,114 "mae_reported": 0.12090394601714284,115 "n": 35,116 "rmse_from_csv": 0.15937419012351917,117 "rmse_reported": 0.15937419012351917118 }119 },120 "passed": true,121 "tolerance": {122 "rel": 1e-05123 }124 },125 {126 "description": "rel_improvement_rmse_h* = 1 - rmse_model/rmse_naive (abs 1e-6)",127 "expected": null,128 "id": "rel_improvement_consistent",129 "observed": {130 "h1": {131 "expected": -0.022905396501212927,132 "reported": -0.022905396501212927133 },134 "h3": {135 "expected": -0.005103249283300526,136 "reported": -0.005103249283300526137 }138 },139 "passed": true,140 "tolerance": {141 "abs": 1e-06142 }143 },144 {145 "description": "data_issues mentions the missing 2025-10 observation",146 "expected": "mention of 2025-10",147 "id": "data_issue_reported",148 "observed": "[{\"issue\": \"Missing observation(s) in UNRATE.csv: ['2025-10-01']\", \"handling\": \"Naive at origin 2025-10-01 uses the 2025-09-01 value; the ARIMA likelihood/filter skips the missing month; (origin,horizon) pairs targeting it are kept in forecasts.csv with empty actual and excluded from error statistic",149 "passed": true,150 "tolerance": null151 }152 ],153 "run_id": "RP0021",154 "schema_error": "",155 "schema_valid": true,156 "scientific_success": true,157 "task_id": "F3-dev-01"158}