{
 "critical_violations": [],
 "evaluated_utc": "2026-10-30T05:33:31.687175+00:00",
 "evaluator_version": "T3-synthesis/1.0",
 "notes": "Deterministic evaluation; reproducibility filled by the harness. Reference = designer extraction with locators + analytical transforms + independent inverse-variance pooling code.",
 "oracle_sha256": "1900db9319ac39a3fb4c4da47dbe8d7be16a596361bf233e529209d362371da0",
 "quality_components": {
  "baseline": 1.0,
  "correctness": 1.0,
  "integrity": 1.0,
  "reporting": 1.0,
  "reproducibility": 1.0
 },
 "quality_score": 1.0,
 "receipt_sha256": "2ba75f14cff03d6d15d6ae78e43a75388f3e6e08d329f7cdffe4c0fb3f595895",
 "required_criteria": [
  {
   "description": "printed values of the five items present in estimate/uncertainty_as_printed",
   "expected": {
    "S1": "Table III Model 2 ExpCon -0.24 (0.14); Section VI-C 95% CI [-0.51, 0.03], p=0.086; n=96",
    "S2": "Section 3: 19% more time (regression S = exp(beta)-1); no numeric CI in the extract (Figure 1 only)",
    "S3": "Table 3 All Content: overall 53.84, treatment 44.29, control 63.10 minutes, -29.79%***; stars only; 181 subjects",
    "S4": "Table 3 Full Corpus: Aware 50.5 s, Control 56.9 s, % time savings 11.1, p-value 0.471; Aware N=56, Control N=52",
    "S5": "Table 1 panel A col (3) Output: Treat x Post -0.278 (0.295), N 18332 (col (2) Release -0.012** (0.005) also accepted as the printed value)"
   },
   "id": "extraction_printed",
   "observed": {
    "S1": {
     "ok": true,
     "text": "ExpCon coefficient -0.24. (Model 2, logged time on task); text: 'effect of AI = 21%' faster, 'about 21%' in abstract SE (0.14); 95% CI = [-0.51, 0.03]; p=0.086 "
    },
    "S2": {
     "ok": true,
     "text": "+19% (allowing AI increases completion time by 19%; 'implement issues in 19% more time on average') none printed (95% CI shown only in Figure 1 / Figure 15, not"
    },
    "S3": {
     "ok": true,
     "text": "All Content: Overall 53.84, Treatment 44.29, Control 63.10 (minutes), Improvement -29.79%*** significance stars only (*** = 0<p<=0.01 per the paper's definition"
    },
    "S4": {
     "ok": true,
     "text": "Full Corpus: Aware Time per Email (s) 50.5, Control Time per Email (s) 56.9, % Time Savings 11.1 p-value 0.471"
    },
    "S5": {
     "ok": true,
     "text": "Treat x Post -0.278 (Output, column 3, panel A) SE (0.295); no stars"
    }
   },
   "passed": true,
   "tolerance": null
  },
  {
   "description": "harmonised theta S1-S4 within tolerance; S5 null",
   "expected": null,
   "id": "harmonised_values",
   "observed": {
    "S1": {
     "expected": -0.24,
     "reported": -0.24
    },
    "S2": {
     "expected": 0.17395330712343798,
     "reported": 0.173953
    },
    "S3": {
     "expected": -0.35382064328549734,
     "reported": -0.353679
    },
    "S4": {
     "expected": -0.11849002415960175,
     "reported": -0.117658
    },
    "S5": {
     "expected": null,
     "reported": null
    }
   },
   "passed": true,
   "tolerance": {
    "S1_est": {
     "abs": 0.001
    },
    "S2_est": {
     "abs": 0.002
    },
    "S3_est": {
     "abs": 0.0015
    },
    "S4_est": {
     "abs": 0.0015
    },
    "pooled_est": {
     "abs": 0.004
    }
   }
  },
  {
   "description": "harmonised SE for S1 and S4 within tolerance; null for S2, S3, S5",
   "expected": null,
   "id": "harmonised_ses",
   "observed": {
    "S1": {
     "expected": 0.14,
     "reported": 0.14
    },
    "S2": {
     "expected": null,
     "reported": null
    },
    "S3": {
     "expected": null,
     "reported": null
    },
    "S4": {
     "expected": 0.16437475344672545,
     "reported": 0.163221
    },
    "S5": {
     "expected": null,
     "reported": null
    }
   },
   "passed": true,
   "tolerance": {
    "S1_se": {
     "abs": 0.0035
    },
    "S4_se": {
     "abs": 0.0025
    },
    "pooled_se": {
     "abs": 0.003
    }
   }
  },
  {
   "description": "harmonisable/poolable flags and k counts equal the reference",
   "expected": {
    "S1": {
     "harmonisable": true,
     "poolable": true
    },
    "S2": {
     "harmonisable": true,
     "poolable": false
    },
    "S3": {
     "harmonisable": true,
     "poolable": false
    },
    "S4": {
     "harmonisable": true,
     "poolable": true
    },
    "S5": {
     "harmonisable": false,
     "poolable": false
    },
    "counts": {
     "k_extracted": 5,
     "k_harmonised": 4,
     "k_pooled": 2
    }
   },
   "id": "flags_counts",
   "observed": {
    "S1": {
     "harmonisable": true,
     "poolable": true
    },
    "S2": {
     "harmonisable": true,
     "poolable": false
    },
    "S3": {
     "harmonisable": true,
     "poolable": false
    },
    "S4": {
     "harmonisable": true,
     "poolable": true
    },
    "S5": {
     "harmonisable": false,
     "poolable": false
    },
    "counts": {
     "k_extracted": 5,
     "k_harmonised": 4,
     "k_pooled": 2
    }
   },
   "passed": true,
   "tolerance": null
  },
  {
   "description": "FE/RE estimates, SEs, tau2, I2, Q within tolerance; CIs and pct change consistent",
   "expected": {
    "fe_estimate": -0.18938,
    "fe_se": 0.10608,
    "i2": 0.0,
    "pooled_pct_change_re": -17.252799241919426,
    "q_stat": 0.319,
    "re_estimate": -0.18938,
    "re_se": 0.10608,
    "tau2": 0.0
   },
   "id": "pooled",
   "observed": {
    "fe_estimate": -0.188143,
    "fe_se": 0.106265,
    "i2": 0.0,
    "pooled_pct_change_re": -17.150405,
    "q_stat": 0.323686,
    "re_estimate": -0.188143,
    "re_se": 0.106265,
    "tau2": 0.0
   },
   "passed": true,
   "tolerance": {
    "i2": {
     "abs": 5.0
    },
    "pooled_est": {
     "abs": 0.004
    },
    "pooled_se": {
     "abs": 0.003
    },
    "q_stat": {
     "abs": 0.05
    },
    "tau2": {
     "abs": 0.002
    }
   }
  },
  {
   "description": "heterogeneity_interpretation = low; pooled_significant_5pct = false",
   "expected": {
    "heterogeneity_interpretation": "low",
    "pooled_significant_5pct": false
   },
   "id": "interpretation",
   "observed": {
    "heterogeneity_interpretation": "low",
    "pooled_significant_5pct": false
   },
   "passed": true,
   "tolerance": null
  },
  {
   "description": "table1.csv (5 rows, required columns) and figure1.png present",
   "expected": null,
   "id": "artifacts",
   "observed": {
    "figure": true,
    "table1_rows": 5
   },
   "passed": true,
   "tolerance": null
  }
 ],
 "run_id": "RT0004",
 "schema_error": "",
 "schema_valid": true,
 "scientific_success": true,
 "task_id": "T3-synthesis"
}