1{2 "critical_violations": [],3 "evaluated_utc": "2026-10-30T05:33:31.687175+00:00",4 "evaluator_version": "T3-synthesis/1.0",5 "notes": "Deterministic evaluation; reproducibility filled by the harness. Reference = designer extraction with locators + analytical transforms + independent inverse-variance pooling code.",6 "oracle_sha256": "1900db9319ac39a3fb4c4da47dbe8d7be16a596361bf233e529209d362371da0",7 "quality_components": {8 "baseline": 1.0,9 "correctness": 1.0,10 "integrity": 1.0,11 "reporting": 1.0,12 "reproducibility": 1.013 },14 "quality_score": 1.0,15 "receipt_sha256": "2ba75f14cff03d6d15d6ae78e43a75388f3e6e08d329f7cdffe4c0fb3f595895",16 "required_criteria": [17 {18 "description": "printed values of the five items present in estimate/uncertainty_as_printed",19 "expected": {20 "S1": "Table III Model 2 ExpCon -0.24 (0.14); Section VI-C 95% CI [-0.51, 0.03], p=0.086; n=96",21 "S2": "Section 3: 19% more time (regression S = exp(beta)-1); no numeric CI in the extract (Figure 1 only)",22 "S3": "Table 3 All Content: overall 53.84, treatment 44.29, control 63.10 minutes, -29.79%***; stars only; 181 subjects",23 "S4": "Table 3 Full Corpus: Aware 50.5 s, Control 56.9 s, % time savings 11.1, p-value 0.471; Aware N=56, Control N=52",24 "S5": "Table 1 panel A col (3) Output: Treat x Post -0.278 (0.295), N 18332 (col (2) Release -0.012** (0.005) also accepted as the printed value)"25 },26 "id": "extraction_printed",27 "observed": {28 "S1": {29 "ok": true,30 "text": "ExpCon coefficient -0.24. (Model 2, logged time on task); text: 'effect of AI = 21%' faster, 'about 21%' in abstract SE (0.14); 95% CI = [-0.51, 0.03]; p=0.086 "31 },32 "S2": {33 "ok": true,34 "text": "+19% (allowing AI increases completion time by 19%; 'implement issues in 19% more time on average') none printed (95% CI shown only in Figure 1 / Figure 15, not"35 },36 "S3": {37 "ok": true,38 "text": "All Content: Overall 53.84, Treatment 44.29, Control 63.10 (minutes), Improvement -29.79%*** significance stars only (*** = 0<p<=0.01 per the paper's definition"39 },40 "S4": {41 "ok": true,42 "text": "Full Corpus: Aware Time per Email (s) 50.5, Control Time per Email (s) 56.9, % Time Savings 11.1 p-value 0.471"43 },44 "S5": {45 "ok": true,46 "text": "Treat x Post -0.278 (Output, column 3, panel A) SE (0.295); no stars"47 }48 },49 "passed": true,50 "tolerance": null51 },52 {53 "description": "harmonised theta S1-S4 within tolerance; S5 null",54 "expected": null,55 "id": "harmonised_values",56 "observed": {57 "S1": {58 "expected": -0.24,59 "reported": -0.2460 },61 "S2": {62 "expected": 0.17395330712343798,63 "reported": 0.17395364 },65 "S3": {66 "expected": -0.35382064328549734,67 "reported": -0.35367968 },69 "S4": {70 "expected": -0.11849002415960175,71 "reported": -0.11765872 },73 "S5": {74 "expected": null,75 "reported": null76 }77 },78 "passed": true,79 "tolerance": {80 "S1_est": {81 "abs": 0.00182 },83 "S2_est": {84 "abs": 0.00285 },86 "S3_est": {87 "abs": 0.001588 },89 "S4_est": {90 "abs": 0.001591 },92 "pooled_est": {93 "abs": 0.00494 }95 }96 },97 {98 "description": "harmonised SE for S1 and S4 within tolerance; null for S2, S3, S5",99 "expected": null,100 "id": "harmonised_ses",101 "observed": {102 "S1": {103 "expected": 0.14,104 "reported": 0.14105 },106 "S2": {107 "expected": null,108 "reported": null109 },110 "S3": {111 "expected": null,112 "reported": null113 },114 "S4": {115 "expected": 0.16437475344672545,116 "reported": 0.163221117 },118 "S5": {119 "expected": null,120 "reported": null121 }122 },123 "passed": true,124 "tolerance": {125 "S1_se": {126 "abs": 0.0035127 },128 "S4_se": {129 "abs": 0.0025130 },131 "pooled_se": {132 "abs": 0.003133 }134 }135 },136 {137 "description": "harmonisable/poolable flags and k counts equal the reference",138 "expected": {139 "S1": {140 "harmonisable": true,141 "poolable": true142 },143 "S2": {144 "harmonisable": true,145 "poolable": false146 },147 "S3": {148 "harmonisable": true,149 "poolable": false150 },151 "S4": {152 "harmonisable": true,153 "poolable": true154 },155 "S5": {156 "harmonisable": false,157 "poolable": false158 },159 "counts": {160 "k_extracted": 5,161 "k_harmonised": 4,162 "k_pooled": 2163 }164 },165 "id": "flags_counts",166 "observed": {167 "S1": {168 "harmonisable": true,169 "poolable": true170 },171 "S2": {172 "harmonisable": true,173 "poolable": false174 },175 "S3": {176 "harmonisable": true,177 "poolable": false178 },179 "S4": {180 "harmonisable": true,181 "poolable": true182 },183 "S5": {184 "harmonisable": false,185 "poolable": false186 },187 "counts": {188 "k_extracted": 5,189 "k_harmonised": 4,190 "k_pooled": 2191 }192 },193 "passed": true,194 "tolerance": null195 },196 {197 "description": "FE/RE estimates, SEs, tau2, I2, Q within tolerance; CIs and pct change consistent",198 "expected": {199 "fe_estimate": -0.18938,200 "fe_se": 0.10608,201 "i2": 0.0,202 "pooled_pct_change_re": -17.252799241919426,203 "q_stat": 0.319,204 "re_estimate": -0.18938,205 "re_se": 0.10608,206 "tau2": 0.0207 },208 "id": "pooled",209 "observed": {210 "fe_estimate": -0.188143,211 "fe_se": 0.106265,212 "i2": 0.0,213 "pooled_pct_change_re": -17.150405,214 "q_stat": 0.323686,215 "re_estimate": -0.188143,216 "re_se": 0.106265,217 "tau2": 0.0218 },219 "passed": true,220 "tolerance": {221 "i2": {222 "abs": 5.0223 },224 "pooled_est": {225 "abs": 0.004226 },227 "pooled_se": {228 "abs": 0.003229 },230 "q_stat": {231 "abs": 0.05232 },233 "tau2": {234 "abs": 0.002235 }236 }237 },238 {239 "description": "heterogeneity_interpretation = low; pooled_significant_5pct = false",240 "expected": {241 "heterogeneity_interpretation": "low",242 "pooled_significant_5pct": false243 },244 "id": "interpretation",245 "observed": {246 "heterogeneity_interpretation": "low",247 "pooled_significant_5pct": false248 },249 "passed": true,250 "tolerance": null251 },252 {253 "description": "table1.csv (5 rows, required columns) and figure1.png present",254 "expected": null,255 "id": "artifacts",256 "observed": {257 "figure": true,258 "table1_rows": 5259 },260 "passed": true,261 "tolerance": null262 }263 ],264 "run_id": "RT0004",265 "schema_error": "",266 "schema_valid": true,267 "scientific_success": true,268 "task_id": "T3-synthesis"269}