1{2 "critical_violations": [],3 "evaluated_utc": "2026-10-30T05:26:36.556678+00:00",4 "evaluator_version": "F4-engine/1.0/F4-conf-01/1.0",5 "notes": "Deterministic oracle-driven evaluation (designer-extracted references with locators). Reproducibility is filled by the harness.",6 "oracle_sha256": "6b1f40035072ca62d8892c68a75946607da2e2816432ecba6604ca0f0bef38e0",7 "quality_components": {8 "baseline": 1.0,9 "correctness": 1.0,10 "integrity": 1.0,11 "reporting": 1.0,12 "reproducibility": 1.013 },14 "quality_score": 1.0,15 "receipt_sha256": "d205bdd6dd6e5cdd47149b10d3e96e0db141d383404649e0467079c94572998a",16 "required_criteria": [17 {18 "description": "Table 6 values exact as printed (ATT 5 km/3 km, SEs, observations 5 km)",19 "expected": {20 "dcr_att_3km": -0.283,21 "dcr_att_3km_se": 0.279,22 "dcr_att_5km": -0.431,23 "dcr_att_5km_se": 0.21,24 "dcr_obs_5km": 1725025 },26 "id": "dcr_table6_values",27 "observed": {28 "dcr_att_3km": -0.283,29 "dcr_att_3km_se": 0.279,30 "dcr_att_5km": -0.431,31 "dcr_att_5km_se": 0.21,32 "dcr_obs_5km": 1725033 },34 "passed": true,35 "tolerance": {36 "dcr_att_3km": {37 "abs": 0.038 },39 "dcr_att_3km_se": {40 "abs": 0.041 },42 "dcr_att_5km": {43 "abs": 0.044 },45 "dcr_att_5km_se": {46 "abs": 0.047 },48 "dcr_obs_5km": {49 "abs": 0.050 }51 }52 },53 {54 "description": "Table 7 values exact as printed (narcotics 3 km, fatal 1 km, HIV 5 km with SEs)",55 "expected": {56 "dcr_fatal_att_1km": -1.207,57 "dcr_fatal_att_1km_se": 0.137,58 "dcr_hiv_att_5km": -0.34,59 "dcr_hiv_att_5km_se": 0.294,60 "dcr_narcotics_att_3km": 0.549,61 "dcr_narcotics_att_3km_se": 0.18762 },63 "id": "dcr_table7_values",64 "observed": {65 "dcr_fatal_att_1km": -1.207,66 "dcr_fatal_att_1km_se": 0.137,67 "dcr_hiv_att_5km": -0.34,68 "dcr_hiv_att_5km_se": 0.294,69 "dcr_narcotics_att_3km": 0.549,70 "dcr_narcotics_att_3km_se": 0.18771 },72 "passed": true,73 "tolerance": {74 "dcr_fatal_att_1km": {75 "abs": 0.076 },77 "dcr_fatal_att_1km_se": {78 "abs": 0.079 },80 "dcr_hiv_att_5km": {81 "abs": 0.082 },83 "dcr_hiv_att_5km_se": {84 "abs": 0.085 },86 "dcr_narcotics_att_3km": {87 "abs": 0.088 },89 "dcr_narcotics_att_3km_se": {90 "abs": 0.091 }92 }93 },94 {95 "description": "RESEA Table 2 values exact as printed (wage ATE + CI, control mean wage, reemployment ATE, weeks ATE + CI with sign)",96 "expected": {97 "ri_control_mean_wage": 33043,98 "ri_reemployment_ate_pp": 1.47,99 "ri_wage_ate": 1153,100 "ri_wage_ate_ci_high": 2141,101 "ri_wage_ate_ci_low": 166,102 "ri_weeks_ui_ate": -1.99,103 "ri_weeks_ui_ate_ci_high": -1.75,104 "ri_weeks_ui_ate_ci_low": -2.24105 },106 "id": "ri_table2_values",107 "observed": {108 "ri_control_mean_wage": 33043,109 "ri_reemployment_ate_pp": 1.47,110 "ri_wage_ate": 1153,111 "ri_wage_ate_ci_high": 2141,112 "ri_wage_ate_ci_low": 166,113 "ri_weeks_ui_ate": -1.99,114 "ri_weeks_ui_ate_ci_high": -1.75,115 "ri_weeks_ui_ate_ci_low": -2.24116 },117 "passed": true,118 "tolerance": {119 "ri_control_mean_wage": {120 "abs": 0.0121 },122 "ri_reemployment_ate_pp": {123 "abs": 0.0124 },125 "ri_wage_ate": {126 "abs": 0.0127 },128 "ri_wage_ate_ci_high": {129 "abs": 0.0130 },131 "ri_wage_ate_ci_low": {132 "abs": 0.0133 },134 "ri_weeks_ui_ate": {135 "abs": 0.0136 },137 "ri_weeks_ui_ate_ci_high": {138 "abs": 0.0139 },140 "ri_weeks_ui_ate_ci_low": {141 "abs": 0.0142 }143 }144 },145 {146 "description": "95% CI of the 5 km ATT derived from the printed SE: -0.431 \u00b1 z\u00b70.210 (z = 1.96 or 2.0 accepted; tolerance 0.02)",147 "expected": {148 "dcr_att_5km_ci_high": -0.01940755999999999,149 "dcr_att_5km_ci_low": -0.84259244150 },151 "id": "dcr_ci_derived",152 "observed": {153 "dcr_att_5km_ci_high": -0.0194,154 "dcr_att_5km_ci_low": -0.8426155 },156 "passed": true,157 "tolerance": {158 "dcr_att_5km_ci_high": {159 "abs": 0.02160 },161 "dcr_att_5km_ci_low": {162 "abs": 0.02163 }164 }165 },166 {167 "description": "SE implied by the wage CI: (2141-166)/(2z) \u2248 503.8 (z = 1.96) or 493.75 (z = 2); tolerance 15",168 "expected": {169 "ri_wage_ate_se_derived": 503.8357847389034170 },171 "id": "ri_se_derived",172 "observed": {173 "ri_wage_ate_se_derived": 503.83174 },175 "passed": true,176 "tolerance": {177 "ri_wage_ate_se_derived": {178 "abs": 15179 }180 }181 },182 {183 "description": "No exact p-value is printed for the HIV 5 km ATT (only the absence of stars) \u2192 null",184 "expected": {185 "dcr_hiv_att_5km_p_value": null186 },187 "id": "hiv_p_null",188 "observed": {189 "dcr_hiv_att_5km_p_value": null190 },191 "passed": true,192 "tolerance": null193 },194 {195 "description": "HIV 5 km ATT carries no star \u2192 not significant at 10%",196 "expected": {197 "dcr_hiv_att_5km_significant_10pct": false198 },199 "id": "hiv_not_sig",200 "observed": {201 "dcr_hiv_att_5km_significant_10pct": false202 },203 "passed": true,204 "tolerance": null205 },206 {207 "description": "The RESEA tables print CIs, not SEs \u2192 ri_wage_ate_se_reported = false",208 "expected": {209 "ri_wage_ate_se_reported": false210 },211 "id": "ri_se_not_reported",212 "observed": {213 "ri_wage_ate_se_reported": false214 },215 "passed": true,216 "tolerance": null217 },218 {219 "description": "locator for dcr_att_5km names source arxiv-2609.27732v1 and the right table/section",220 "expected": "(?s)(?=.*arxiv-2609\\.27732v1)(?=.*(Table\\s*6))",221 "id": "loc_dcr_att_5km",222 "observed": {223 "locators.dcr_att_5km": "arxiv-2609.27732v1 | Table 6 | row \"ATT\" | column \"5 km\""224 },225 "passed": true,226 "tolerance": null227 },228 {229 "description": "locator for dcr_att_5km_se names source arxiv-2609.27732v1 and the right table/section",230 "expected": "(?s)(?=.*arxiv-2609\\.27732v1)(?=.*(Table\\s*6))",231 "id": "loc_dcr_att_5km_se",232 "observed": {233 "locators.dcr_att_5km_se": "arxiv-2609.27732v1 | Table 6 | row \"ATT\" | column \"5 km\" (standard error in parentheses on the line below the coefficient)"234 },235 "passed": true,236 "tolerance": null237 },238 {239 "description": "locator for dcr_att_3km names source arxiv-2609.27732v1 and the right table/section",240 "expected": "(?s)(?=.*arxiv-2609\\.27732v1)(?=.*(Table\\s*6))",241 "id": "loc_dcr_att_3km",242 "observed": {243 "locators.dcr_att_3km": "arxiv-2609.27732v1 | Table 6 | row \"ATT\" | column \"3 km\""244 },245 "passed": true,246 "tolerance": null247 },248 {249 "description": "locator for dcr_att_3km_se names source arxiv-2609.27732v1 and the right table/section",250 "expected": "(?s)(?=.*arxiv-2609\\.27732v1)(?=.*(Table\\s*6))",251 "id": "loc_dcr_att_3km_se",252 "observed": {253 "locators.dcr_att_3km_se": "arxiv-2609.27732v1 | Table 6 | row \"ATT\" | column \"3 km\" (standard error in parentheses on the line below the coefficient)"254 },255 "passed": true,256 "tolerance": null257 },258 {259 "description": "locator for dcr_obs_5km names source arxiv-2609.27732v1 and the right table/section",260 "expected": "(?s)(?=.*arxiv-2609\\.27732v1)(?=.*(Table\\s*6))",261 "id": "loc_dcr_obs_5km",262 "observed": {263 "locators.dcr_obs_5km": "arxiv-2609.27732v1 | Table 6 | row \"Observations\" | column \"5 km\""264 },265 "passed": true,266 "tolerance": null267 },268 {269 "description": "locator for dcr_narcotics_att_3km names source arxiv-2609.27732v1 and the right table/section",270 "expected": "(?s)(?=.*arxiv-2609\\.27732v1)(?=.*(Table\\s*7))",271 "id": "loc_dcr_narcotics_att_3km",272 "observed": {273 "locators.dcr_narcotics_att_3km": "arxiv-2609.27732v1 | Table 7 | row \"Narcotics\" | column \"3 km\""274 },275 "passed": true,276 "tolerance": null277 },278 {279 "description": "locator for dcr_narcotics_att_3km_se names source arxiv-2609.27732v1 and the right table/section",280 "expected": "(?s)(?=.*arxiv-2609\\.27732v1)(?=.*(Table\\s*7))",281 "id": "loc_dcr_narcotics_att_3km_se",282 "observed": {283 "locators.dcr_narcotics_att_3km_se": "arxiv-2609.27732v1 | Table 7 | row \"Narcotics\" | column \"3 km\" (standard error in parentheses on the line below the coefficient)"284 },285 "passed": true,286 "tolerance": null287 },288 {289 "description": "locator for dcr_fatal_att_1km names source arxiv-2609.27732v1 and the right table/section",290 "expected": "(?s)(?=.*arxiv-2609\\.27732v1)(?=.*(Table\\s*7))",291 "id": "loc_dcr_fatal_att_1km",292 "observed": {293 "locators.dcr_fatal_att_1km": "arxiv-2609.27732v1 | Table 7 | row \"Drug-related (fatal)\" | column \"1 km\""294 },295 "passed": true,296 "tolerance": null297 },298 {299 "description": "locator for dcr_fatal_att_1km_se names source arxiv-2609.27732v1 and the right table/section",300 "expected": "(?s)(?=.*arxiv-2609\\.27732v1)(?=.*(Table\\s*7))",301 "id": "loc_dcr_fatal_att_1km_se",302 "observed": {303 "locators.dcr_fatal_att_1km_se": "arxiv-2609.27732v1 | Table 7 | row \"Drug-related (fatal)\" | column \"1 km\" (standard error in parentheses on the line below the coefficient)"304 },305 "passed": true,306 "tolerance": null307 },308 {309 "description": "locator for dcr_hiv_att_5km names source arxiv-2609.27732v1 and the right table/section",310 "expected": "(?s)(?=.*arxiv-2609\\.27732v1)(?=.*(Table\\s*7))",311 "id": "loc_dcr_hiv_att_5km",312 "observed": {313 "locators.dcr_hiv_att_5km": "arxiv-2609.27732v1 | Table 7 | row \"HIV\" | column \"5 km\""314 },315 "passed": true,316 "tolerance": null317 },318 {319 "description": "locator for dcr_hiv_att_5km_se names source arxiv-2609.27732v1 and the right table/section",320 "expected": "(?s)(?=.*arxiv-2609\\.27732v1)(?=.*(Table\\s*7))",321 "id": "loc_dcr_hiv_att_5km_se",322 "observed": {323 "locators.dcr_hiv_att_5km_se": "arxiv-2609.27732v1 | Table 7 | row \"HIV\" | column \"5 km\" (standard error in parentheses on the line below the coefficient)"324 },325 "passed": true,326 "tolerance": null327 },328 {329 "description": "locator for dcr_hiv_att_5km_p_value names source arxiv-2609.27732v1 and the right table/section",330 "expected": "(?s)(?=.*arxiv-2609\\.27732v1)(?=.*(not (reported|printed|given|available)|null|only (stars|significance)))",331 "id": "loc_dcr_hiv_att_5km_p_value",332 "observed": {333 "locators.dcr_hiv_att_5km_p_value": "not reported in arxiv-2609.27732v1 (Table 7 prints only significance stars; cell row \"HIV\" column \"5 km\" = \"-0.340\" carries no star)"334 },335 "passed": true,336 "tolerance": null337 },338 {339 "description": "locator for dcr_att_5km_ci_low names source arxiv-2609.27732v1 and the right table/section",340 "expected": "(?s)(?=.*arxiv-2609\\.27732v1)(?=.*(Table\\s*6.*deriv|deriv.*Table\\s*6))",341 "id": "loc_dcr_att_5km_ci_low",342 "observed": {343 "locators.dcr_att_5km_ci_low": "arxiv-2609.27732v1 | Table 6 | row \"ATT\" | column \"5 km\" | derived: estimate -/+ 1.96 x printed SE"344 },345 "passed": true,346 "tolerance": null347 },348 {349 "description": "locator for dcr_att_5km_ci_high names source arxiv-2609.27732v1 and the right table/section",350 "expected": "(?s)(?=.*arxiv-2609\\.27732v1)(?=.*(Table\\s*6.*deriv|deriv.*Table\\s*6))",351 "id": "loc_dcr_att_5km_ci_high",352 "observed": {353 "locators.dcr_att_5km_ci_high": "arxiv-2609.27732v1 | Table 6 | row \"ATT\" | column \"5 km\" | derived: estimate -/+ 1.96 x printed SE"354 },355 "passed": true,356 "tolerance": null357 },358 {359 "description": "locator for ri_wage_ate names source arxiv-2606.14621v1 and the right table/section",360 "expected": "(?s)(?=.*arxiv-2606\\.14621v1)(?=.*(Table\\s*[23]))",361 "id": "loc_ri_wage_ate",362 "observed": {363 "locators.ri_wage_ate": "arxiv-2606.14621v1 | Table 2 | row \"Wage\" | column \"Average treatment effect estimate\" (point estimate)"364 },365 "passed": true,366 "tolerance": null367 },368 {369 "description": "locator for ri_wage_ate_ci_low names source arxiv-2606.14621v1 and the right table/section",370 "expected": "(?s)(?=.*arxiv-2606\\.14621v1)(?=.*(Table\\s*[23]))",371 "id": "loc_ri_wage_ate_ci_low",372 "observed": {373 "locators.ri_wage_ate_ci_low": "arxiv-2606.14621v1 | Table 2 | row \"Wage\" | column \"Average treatment effect estimate\" (95% CI lower bound)"374 },375 "passed": true,376 "tolerance": null377 },378 {379 "description": "locator for ri_wage_ate_ci_high names source arxiv-2606.14621v1 and the right table/section",380 "expected": "(?s)(?=.*arxiv-2606\\.14621v1)(?=.*(Table\\s*[23]))",381 "id": "loc_ri_wage_ate_ci_high",382 "observed": {383 "locators.ri_wage_ate_ci_high": "arxiv-2606.14621v1 | Table 2 | row \"Wage\" | column \"Average treatment effect estimate\" (95% CI upper bound)"384 },385 "passed": true,386 "tolerance": null387 },388 {389 "description": "locator for ri_wage_ate_se_derived names source arxiv-2606.14621v1 and the right table/section",390 "expected": "(?s)(?=.*arxiv-2606\\.14621v1)(?=.*(Table\\s*[23].*deriv|deriv.*Table\\s*[23]))",391 "id": "loc_ri_wage_ate_se_derived",392 "observed": {393 "locators.ri_wage_ate_se_derived": "arxiv-2606.14621v1 | Table 2 | row \"Wage\" | column \"Average treatment effect estimate\" | derived: (CI upper - CI lower)/(2 x 1.96); SE not reported in arxiv-2606.14621v1"394 },395 "passed": true,396 "tolerance": null397 },398 {399 "description": "locator for ri_control_mean_wage names source arxiv-2606.14621v1 and the right table/section",400 "expected": "(?s)(?=.*arxiv-2606\\.14621v1)(?=.*(Table\\s*2|Results|Section\\s*3))",401 "id": "loc_ri_control_mean_wage",402 "observed": {403 "locators.ri_control_mean_wage": "arxiv-2606.14621v1 | Table 2 | row \"Wage\" | column \"Control\" (mean; SD in parentheses not used)"404 },405 "passed": true,406 "tolerance": null407 },408 {409 "description": "locator for ri_reemployment_ate_pp names source arxiv-2606.14621v1 and the right table/section",410 "expected": "(?s)(?=.*arxiv-2606\\.14621v1)(?=.*(Table\\s*[23]))",411 "id": "loc_ri_reemployment_ate_pp",412 "observed": {413 "locators.ri_reemployment_ate_pp": "arxiv-2606.14621v1 | Table 2 | row \"Reemployment\" | column \"Average treatment effect estimate\" (point estimate, pp)"414 },415 "passed": true,416 "tolerance": null417 },418 {419 "description": "locator for ri_weeks_ui_ate names source arxiv-2606.14621v1 and the right table/section",420 "expected": "(?s)(?=.*arxiv-2606\\.14621v1)(?=.*(Table\\s*[23]))",421 "id": "loc_ri_weeks_ui_ate",422 "observed": {423 "locators.ri_weeks_ui_ate": "arxiv-2606.14621v1 | Table 2 | row \"Weeks on UI\" | column \"Average treatment effect estimate\" (point estimate, weeks)"424 },425 "passed": true,426 "tolerance": null427 },428 {429 "description": "locator for ri_weeks_ui_ate_ci_low names source arxiv-2606.14621v1 and the right table/section",430 "expected": "(?s)(?=.*arxiv-2606\\.14621v1)(?=.*(Table\\s*[23]))",431 "id": "loc_ri_weeks_ui_ate_ci_low",432 "observed": {433 "locators.ri_weeks_ui_ate_ci_low": "arxiv-2606.14621v1 | Table 2 | row \"Weeks on UI\" | column \"Average treatment effect estimate\" (95% CI lower bound)"434 },435 "passed": true,436 "tolerance": null437 },438 {439 "description": "locator for ri_weeks_ui_ate_ci_high names source arxiv-2606.14621v1 and the right table/section",440 "expected": "(?s)(?=.*arxiv-2606\\.14621v1)(?=.*(Table\\s*[23]))",441 "id": "loc_ri_weeks_ui_ate_ci_high",442 "observed": {443 "locators.ri_weeks_ui_ate_ci_high": "arxiv-2606.14621v1 | Table 2 | row \"Weeks on UI\" | column \"Average treatment effect estimate\" (95% CI upper bound)"444 },445 "passed": true,446 "tolerance": null447 }448 ],449 "run_id": "RC0285",450 "schema_error": "",451 "schema_valid": true,452 "scientific_success": true,453 "task_id": "F4-conf-01"454}