{
 "task_id": "T3-synthesis",
 "target_definition": "theta = ln(T_AI / T_noAI): natural log of the ratio of mean task-completion time with generative-AI assistance to mean completion time without it, for the same tasks, from a randomised comparison (negative = faster with AI).",
 "items": {
  "S1": {
   "estimate_as_printed": "-0.24. (beta_1 for ExpCon, Model 2, Table III; Section VI-C: beta_1=-0.24, i.e. 21% faster)",
   "uncertainty_as_printed": "SE (0.14); 95% CI = [-0.51, 0.03], p=0.086 (NS)",
   "sample_as_printed": "n = 96 (Table III caption); Section VI-B: N=47 AI, N=46 no-AI",
   "definition": "OLS coefficient on experimental condition (AI exposure) with natural-log time on task (LogToT) as dependent variable, developer-level covariates (Model 2), one enterprise coding task, randomised",
   "locator": "arxiv-2410.12944v3, Section VI-C Table III, row ExpCon, column (2) 'Dev' (second value on collapsed line L76; SE line L78); CI and p in Section VI-C text",
   "harmonisable": true,
   "harmonised_estimate": -0.24,
   "harmonised_se": 0.14,
   "poolable": true,
   "transformation": "log-time coefficient: theta = b = -0.24; SE(theta) = SE(b) = 0.14 (printed). Check: SE from CI = (0.03-(-0.51))/(2*1.96) = 0.1378",
   "reason_if_not": null
  },
  "S2": {
   "estimate_as_printed": "+19% (allowing AI increases completion time by 19%; regression-based, Section 3 / Section D.1)",
   "uncertainty_as_printed": "none printed (95% CI shown only in Figure 1 / Figure 15, not reproduced in the extract)",
   "sample_as_printed": "16 developers, 246 tasks; 136 issues AI-allowed, 110 AI-disallowed",
   "definition": "S = E[time AI allowed]/E[time AI disallowed] - 1, estimated as exp(beta)-1 from OLS of log implementation time on AI indicator and log forecast time; issue-level randomisation",
   "locator": "arxiv-2507.09089v2, Abstract and Section 3 'Results' (opening paragraph); estimator Sections 2.3, D.1-D.3",
   "harmonisable": true,
   "harmonised_estimate": 0.17395330712343798,
   "harmonised_se": null,
   "poolable": false,
   "transformation": "percentage change in time: theta = ln(1 + c) = ln(1 + 0.19) = 0.173953",
   "reason_if_not": "No SE, CI or p-value printed in the extract; the CI exists only in a figure that is not reproduced, so SE(theta) cannot be obtained (harmonisable, not poolable)."
  },
  "S3": {
   "estimate_as_printed": "Overall 53.84, Treatment 44.29, Control 63.10 (minutes); Improvement -29.79%***",
   "uncertainty_as_printed": "significance stars only: *** (paper's definition 0<p<=0.01); no SE or CI printed",
   "sample_as_printed": "181 subjects (randomised half treatment, half control)",
   "definition": "difference in mean task duration (minutes) between Copilot and control groups across all tasks, holding accuracy constant (control duration predicted at treatment accuracy, Appendix B), randomised",
   "locator": "arxiv-2411.01067v2, Section 4.2 Table 3, 'Aggregate Results' block, row 'All Content', columns Treatment/Control/Improvement",
   "harmonisable": true,
   "harmonised_estimate": -0.353679434958934,
   "harmonised_se": null,
   "poolable": false,
   "transformation": "percentage change in time: theta = ln(1 + c) = ln(1 + (-0.2979)) = -0.353679 (cf. ln(44.29/63.10) = -0.353962)",
   "reason_if_not": "Only significance stars (p<=0.01) are printed; a p-value threshold does not identify SE(theta) (harmonisable, not poolable)."
  },
  "S4": {
   "estimate_as_printed": "Aware 50.5 s, Control 56.9 s per email; % Time Savings 11.1",
   "uncertainty_as_printed": "p-value 0.471",
   "sample_as_printed": "Aware N=56, Control N=52 (167 analysts across three arms incl. Blind N=59)",
   "definition": "percentage time saving in mean time per email over the full 25-email queue, Aware (agent verdicts/rationale + prioritised queue) vs Control (no AI), randomised arms",
   "locator": "arxiv-2511.13860v1, Section 3.3 Table 3 'Comparison of Email Processing Times', column 'Full Corpus', rows Aware/Control time, % Time Savings, p-value",
   "harmonisable": true,
   "harmonised_estimate": -0.11765804346823246,
   "harmonised_se": 0.1632205919720678,
   "poolable": true,
   "transformation": "percentage change in time c = -11.1/100 = -0.111; theta = ln(1 + c) = -0.117658; approximate SE from two-sided p: SE = |theta|/z(1-0.471/2) = 0.117658/0.720853 = 0.163221 (approximate SE)",
   "reason_if_not": null
  },
  "S5": {
   "estimate_as_printed": "-0.278 (Treat x Post, Output, column 3, panel A)",
   "uncertainty_as_printed": "SE (0.295) (clustered at user level); no significance star",
   "sample_as_printed": "N = 18332 (user-day observations, users with affiliation)",
   "definition": "difference-in-differences coefficient on aggregate GitHub Output measure (sum of Push, PR, PR comments, commit comments, Create, Issues events) for Italian vs Austrian/French users, pre 27/28.03 vs post 03/04.04.2023 around Italy's ChatGPT ban",
   "locator": "arxiv-2304.09339v1, Section 4 Table 1, panel A, row Treat x Post, column (3) Output (Users with Affiliation)",
   "harmonisable": false,
   "harmonised_estimate": null,
   "harmonised_se": null,
   "poolable": false,
   "transformation": "none applied (not harmonisable)",
   "reason_if_not": "Effect on output counts (GitHub events) from a DiD around withdrawal of access, not a comparison of task-completion times with vs without AI on the same tasks, and not randomised."
  }
 },
 "k_extracted": 5,
 "k_harmonised": 4,
 "k_pooled": 2,
 "pooled_studies": [
  "S1",
  "S4"
 ],
 "fe_estimate": -0.18814332871148873,
 "fe_se": 0.10626483095451474,
 "fe_ci_low": -0.39642239738233764,
 "fe_ci_high": 0.02013573995936016,
 "re_estimate": -0.18814332871148873,
 "re_se": 0.10626483095451474,
 "re_ci_low": -0.39642239738233764,
 "re_ci_high": 0.02013573995936016,
 "q_stat": 0.3236860522786328,
 "q_df": 1,
 "q_pvalue": 0.5694008376973132,
 "i2": 0.0,
 "tau2": 0.0,
 "heterogeneity_interpretation": "low",
 "pooled_significant_5pct": false,
 "pooled_pct_change_re": -17.15040485348688,
 "pooled_pct_change_re_ci": [
  -32.72775202989503,
  2.0339831514392115
 ],
 "weights_fe": {
  "S1": 0.5761333825403874,
  "S4": 0.4238666174596126
 },
 "weights_re": {
  "S1": 0.5761333825403874,
  "S4": 0.4238666174596126
 },
 "baseline_unweighted_mean_theta_harmonised": -0.13434604282593213,
 "baseline_unweighted_mean_theta_poolable": -0.17882902173411622,
 "sensitivity_s1_se_from_ci": {
  "s1_se": 0.13775510204081634,
  "re_estimate": -0.1891067723193802,
  "re_se": 0.10527305720065441,
  "re_ci_low": -0.39544196443266283,
  "re_ci_high": 0.01722841979390244,
  "q_stat": 0.3281104262022853,
  "i2": 0.0,
  "tau2": 0.0
 },
 "feasibility": "complete",
 "feasibility_reason": null,
 "data_issues": [
  {
   "issue": "S1: Section VI-B reports N=47 (AI) + N=46 (no-AI) = 93 developers, while Table III states n = 96.",
   "handling": "Not resolvable from the extract; reported. sample_as_printed lists both; does not affect theta or SE, which are taken from Table III/Section VI-C."
  },
  {
   "issue": "S1: Section VI-B gives Mean LogToT=4.6 for the AI group and 4.3 for no-AI while stating AI was faster (96 vs 114 min); the log means appear swapped/inconsistent.",
   "handling": "These log means are not used."
  },
  {
   "issue": "S1: Table III is rendered with the four model columns collapsed into one line; SE (0.14) is rounded, and the CI [-0.51, 0.03] implies SE 0.1378.",
   "handling": "Model 2 taken as the second value per the editorial note; printed SE 0.14 used; CI-based SE reported as sensitivity (sensitivity_s1_se_from_ci)."
  },
  {
   "issue": "S2: The uncertainty of the headline +19% estimate is shown only in Figure 1/Figure 15, which are not reproduced; no SE or p-value printed.",
   "handling": "Harmonised (theta = ln 1.19) but not poolable; harmonised_se = null."
  },
  {
   "issue": "S3: Treatment/Control means 44.29/63.10 imply -29.81%, printed improvement is -29.79% (slightly beyond rounding of the printed means); significance shown only by stars with an unusual convention (* = 0.05<p<=0.1).",
   "handling": "Printed -29.79% used as instructed (difference in theta ~0.0003); not poolable because stars do not identify an SE."
  },
  {
   "issue": "S4: Table 3 units ambiguous: Malicious/Non-Malicious times per email (1.8-2.8 's') cannot average to Full Corpus times (50.5/56.9 s) under any common unit; Full Corpus ratio 50.5/56.9 implies 11.25% saving vs printed 11.1% (within rounding).",
   "handling": "Only the Full Corpus column is used; theta depends only on the printed % saving (unit-free). Flagged as a source inconsistency."
  },
  {
   "issue": "S4: SE is approximate, back-calculated from a two-sided p-value (0.471) under a normal approximation on the log scale, although the paper's test is presumably on the time difference.",
   "handling": "Marked as approximate SE in transformation; pooled results carry this caveat."
  },
  {
   "issue": "S5: Table 1 note prints '** p<0.5' (presumably p<0.05); row label rendered as '\\times TreatPost'; panel B header lists 11.04 although the text says it was excluded; rendered date line 'August 24, 2026' is an artefact.",
   "handling": "No effect on the extracted panel A column 3 value; S5 is not harmonisable anyway."
  }
 ],
 "n_input_rows": {
  "studies.json": 5,
  "README.md": 17,
  "sources/arxiv-2410.12944v3.txt": 112,
  "sources/arxiv-2507.09089v2.txt": 132,
  "sources/arxiv-2411.01067v2.txt": 109,
  "sources/arxiv-2511.13860v1.txt": 162,
  "sources/arxiv-2304.09339v1.txt": 108
 },
 "n_analysed": 5,
 "random_seed": null
}