1{2 "critical_violations": [],3 "evaluated_utc": "2026-10-30T04:18:39.650957+00:00",4 "evaluator_version": "F3-conf-02/1.0",5 "notes": "Deterministic evaluation; reproducibility filled by the harness. Vintage discipline is verified from train_last_date/train_last_value/n_train and the naive rows.",6 "oracle_sha256": "eedec7d03dd243b4193827a51afdac386dae16fd206fe289174b0f9fb316ba82",7 "quality_components": {8 "baseline": 0.5,9 "correctness": 0.42857142857142855,10 "integrity": 1.0,11 "reporting": 1.0,12 "reproducibility": 1.013 },14 "quality_score": 0.7214,15 "receipt_sha256": "328d4695c99ecdce440976dcd32ec06927e082f328b0814921675e2ccc402d5d",16 "required_criteria": [17 {18 "description": "Naive RMSE/MAE at h=1 (last observation of the origin vintage vs first-release actual) exact, rel 1e-6",19 "expected": {20 "mae_naive_h1": 1.6874999999999996,21 "rmse_naive_h1": 2.010286049297462422 },23 "id": "naive_h1",24 "observed": {25 "mae_naive_h1": 1.6875,26 "rmse_naive_h1": 2.010327 },28 "passed": false,29 "tolerance": {30 "mae_naive_h1": {31 "rel": 1e-0632 },33 "rmse_naive_h1": {34 "rel": 1e-0635 }36 }37 },38 {39 "description": "Naive RMSE/MAE at h=2 exact, rel 1e-6",40 "expected": {41 "mae_naive_h2": 1.8124999999999998,42 "rmse_naive_h2": 2.26522625801485943 },44 "id": "naive_h2",45 "observed": {46 "mae_naive_h2": 1.8125,47 "rmse_naive_h2": 2.265248 },49 "passed": false,50 "tolerance": {51 "mae_naive_h2": {52 "rel": 1e-0653 },54 "rmse_naive_h2": {55 "rel": 1e-0656 }57 }58 },59 {60 "description": "mean_abs_revision_h1_targets = mean |current vintage - first release| over the 8 h=1 targets, rel 1e-6",61 "expected": {62 "mean_abs_revision_h1_targets": 0.562500000000000163 },64 "id": "revision",65 "observed": {66 "mean_abs_revision_h1_targets": 0.562567 },68 "passed": true,69 "tolerance": {70 "mean_abs_revision_h1_targets": {71 "rel": 1e-0672 }73 }74 },75 {76 "description": "n_origins = 8",77 "expected": {78 "n_origins": 879 },80 "id": "counts",81 "observed": {82 "n_origins": 883 },84 "passed": true,85 "tolerance": {86 "n_origins": {87 "abs": 088 }89 }90 },91 {92 "description": "forecasts.csv: 32 rows, vintage = origin, targets = last vintage quarter + h, training data = that vintage, naive = last vintage value, actual = first release",93 "expected": {94 "n_rows": 3295 },96 "id": "forecasts_chronology",97 "observed": {98 "actual_mismatches": 0,99 "bad_rows": 0,100 "model_rows": 16,101 "n_rows": 32,102 "naive_rows": 16103 },104 "passed": true,105 "tolerance": null106 },107 {108 "description": "model RMSE/MAE agree with forecasts.csv (rel 1e-5)",109 "expected": null,110 "id": "model_errors_consistent",111 "observed": {112 "h1": {113 "mae_from_csv": 1.2336553749999999,114 "mae_reported": 1.2337,115 "rmse_from_csv": 1.4854812413428937,116 "rmse_reported": 1.4855117 },118 "h2": {119 "mae_from_csv": 1.029787375,120 "mae_reported": 1.0298,121 "rmse_from_csv": 1.4000729511988383,122 "rmse_reported": 1.4001123 }124 },125 "passed": false,126 "tolerance": {127 "rel": 1e-05128 }129 },130 {131 "description": "rel_improvement = 1 - rmse_model/rmse_naive (abs 1e-6)",132 "expected": null,133 "id": "rel_improvement_consistent",134 "observed": {135 "h1": {136 "expected": 0.2610555638461921,137 "reported": 0.2611138 },139 "h2": {140 "expected": 0.38190888221790575,141 "reported": 0.3819142 }143 },144 "passed": false,145 "tolerance": {146 "abs": 1e-06147 }148 }149 ],150 "run_id": "RC0063",151 "schema_error": "",152 "schema_valid": true,153 "scientific_success": false,154 "task_id": "F3-conf-02"155}