1{2 "critical_violations": [],3 "evaluated_utc": "2026-10-30T05:36:33.812656+00:00",4 "evaluator_version": "F4-engine/1.0/F4-conf-04/1.0",5 "notes": "Deterministic oracle-driven evaluation (designer-extracted references with locators). Reproducibility is filled by the harness.",6 "oracle_sha256": "6eb80dbbab7f408a8db45bef65c2a569fc1523d30f1976ec112f63c5bd621099",7 "quality_components": {8 "baseline": 1.0,9 "correctness": 1.0,10 "integrity": 1.0,11 "reporting": 1.0,12 "reproducibility": 1.013 },14 "quality_score": 1.0,15 "receipt_sha256": "07e883f7d1d9b47fd8ba08f3202f59a483be5146dbf6d7fe21d4e6d01010d4c4",16 "required_criteria": [17 {18 "description": "unverifiable_references == ['R3'] (title, year and finding all disagree with the source)",19 "expected": {20 "unverifiable_references": [21 "R3"22 ]23 },24 "id": "unverifiable_set",25 "observed": {26 "unverifiable_references": [27 "R3"28 ]29 },30 "passed": true,31 "tolerance": null32 },33 {34 "description": "finding_supported true for R1, R2, R4, R5 and false for R3",35 "expected": {36 "reference_checks.R1.finding_supported": true,37 "reference_checks.R2.finding_supported": true,38 "reference_checks.R3.finding_supported": false,39 "reference_checks.R4.finding_supported": true,40 "reference_checks.R5.finding_supported": true41 },42 "id": "finding_flags",43 "observed": {44 "reference_checks.R1.finding_supported": true,45 "reference_checks.R2.finding_supported": true,46 "reference_checks.R3.finding_supported": false,47 "reference_checks.R4.finding_supported": true,48 "reference_checks.R5.finding_supported": true49 },50 "passed": true,51 "tolerance": null52 },53 {54 "description": "title_matches true for R1, R2, R4, R5 and false for R3",55 "expected": {56 "reference_checks.R1.title_matches": true,57 "reference_checks.R2.title_matches": true,58 "reference_checks.R3.title_matches": false,59 "reference_checks.R4.title_matches": true,60 "reference_checks.R5.title_matches": true61 },62 "id": "title_flags",63 "observed": {64 "reference_checks.R1.title_matches": true,65 "reference_checks.R2.title_matches": true,66 "reference_checks.R3.title_matches": false,67 "reference_checks.R4.title_matches": true,68 "reference_checks.R5.title_matches": true69 },70 "passed": true,71 "tolerance": null72 },73 {74 "description": "year_matches false for R3 (2024 vs 2026) and true for the others (R4: 2025 = first submission year)",75 "expected": {76 "reference_checks.R1.year_matches": true,77 "reference_checks.R2.year_matches": true,78 "reference_checks.R3.year_matches": false,79 "reference_checks.R4.year_matches": true,80 "reference_checks.R5.year_matches": true81 },82 "id": "year_flags",83 "observed": {84 "reference_checks.R1.year_matches": true,85 "reference_checks.R2.year_matches": true,86 "reference_checks.R3.year_matches": false,87 "reference_checks.R4.year_matches": true,88 "reference_checks.R5.year_matches": true89 },90 "passed": true,91 "tolerance": null92 },93 {94 "description": "n_verified = 4",95 "expected": {96 "n_verified": 497 },98 "id": "n_verified",99 "observed": {100 "n_verified": 4101 },102 "passed": true,103 "tolerance": {104 "n_verified": {105 "abs": 0106 }107 }108 },109 {110 "description": "locator for R1 names arxiv-2609.27732v1 and a deciding table/section",111 "expected": "(?s)(?=.*arxiv-2609\\.27732v1)(?=.*(Table\\s*6|Abstract))",112 "id": "loc_R1",113 "observed": {114 "reference_checks.R1.locator": "arxiv-2609.27732v1 | Table 6, row 'ATT' (and SE row), column '5 km'; abstract"115 },116 "passed": true,117 "tolerance": null118 },119 {120 "description": "locator for R2 names arxiv-2606.14621v1 and a deciding table/section",121 "expected": "(?s)(?=.*arxiv-2606\\.14621v1)(?=.*(Table\\s*[23]|Abstract|Section\\s*3|Results))",122 "id": "loc_R2",123 "observed": {124 "reference_checks.R2.locator": "arxiv-2606.14621v1 | Table 2, column 'Average treatment effect estimate', rows Wage / Reemployment / Weeks on UI; abstract (N, preregistered RCT)"125 },126 "passed": true,127 "tolerance": null128 },129 {130 "description": "locator for R3 names arxiv-2603.03260v2 and a deciding table/section",131 "expected": "(?s)(?=.*arxiv-2603\\.03260v2)(?=.*(Table\\s*3|Section\\s*3|Abstract|Section\\s*5|Conclusion|Figure\\s*2))",132 "id": "loc_R3",133 "observed": {134 "reference_checks.R3.locator": "arxiv-2603.03260v2 | Section 3.2 'Headline dynamic pattern' (stacked coefficient 0.005 at tau=0); Table 3, row 'Clean-onset threshold-2', columns tau=4 (0.037) and tau=5 (0.034)"135 },136 "passed": true,137 "tolerance": null138 },139 {140 "description": "locator for R4 names arxiv-2505.04555v3 and a deciding table/section",141 "expected": "(?s)(?=.*arxiv-2505\\.04555v3)(?=.*(Table\\s*2|Appendix))",142 "id": "loc_R4",143 "observed": {144 "reference_checks.R4.locator": "arxiv-2505.04555v3 | Appendix A.1, Table 2, rows 'Employment elasticity w.r.t MW' and 'Emp. elasticity w.r.t affected wage' (Estimate (SE) column); abstract (Japan's spot labor market)"145 },146 "passed": true,147 "tolerance": null148 },149 {150 "description": "locator for R5 names arxiv-2510.03658v1 and a deciding table/section",151 "expected": "(?s)(?=.*arxiv-2510\\.03658v1)(?=.*(Table\\s*3|Section\\s*4))",152 "id": "loc_R5",153 "observed": {154 "reference_checks.R5.locator": "arxiv-2510.03658v1 | Table 3, row 'Treatment' (and SE row), '% Change' and 'Mean Dep. Var.', column (4); Section 4.1 (column 4 is preferred)"155 },156 "passed": true,157 "tolerance": null158 }159 ],160 "run_id": "RA0014",161 "schema_error": "",162 "schema_valid": true,163 "scientific_success": true,164 "task_id": "F4-conf-04"165}