{
 "task_id": "T3-synthesis",
 "target_definition": "theta = ln(T_AI / T_noAI): natural log of the ratio of mean task-completion time with access to a generative-AI assistant to mean completion time without it, for the same tasks, from a randomised comparison; negative values mean faster with AI.",
 "items": {
  "S1": {
   "estimate_as_printed": "-0.24 (ExpCon, Model 2, LogToT; marker '.'); text: beta_1=-0.24, effect of AI = 21%",
   "uncertainty_as_printed": "SE (0.14); 95% CI = [-0.51, 0.03]; p=0.086",
   "sample_as_printed": "Table III: n = 96 (Section VI-B: N=47 AI, N=46 no-AI)",
   "definition": "OLS coefficient of the experimental condition (AI vs no-AI, randomised) on the natural log of time on task, controlling for developer-level factors (Model 2)",
   "locator": "arxiv-2410.12944v3; Table III (L76-L78), row ExpCon, column (2) 'Dev'; Section VI-C (L58) for CI and p",
   "harmonisable": true,
   "harmonised_estimate": -0.24,
   "harmonised_se": 0.14,
   "poolable": true,
   "transformation": "log-time coefficient: theta = b = -0.24; SE(theta) = SE(b) = 0.14 (printed SE used; CI-based SE = (0.03 - (-0.51))/3.92 = 0.1378 used only as sensitivity)",
   "reason_if_not": null
  },
  "S2": {
   "estimate_as_printed": "19% more time on average (AI allowed increases completion time by 19%)",
   "uncertainty_as_printed": "none printed (confidence interval shown only in Figure 1 / Figure 15, not reproduced in the extract)",
   "sample_as_printed": "16 developers, 246 tasks; 136 issues AI-allowed, 110 issues AI-disallowed",
   "definition": "S = E[time with AI allowed]/E[time AI disallowed] - 1, estimated as exp(beta)-1 from OLS of log implementation time on AI indicator and log forecast time (issue-level randomisation)",
   "locator": "arxiv-2507.09089v2; Section 3 Results (L62); abstract (L34); estimator in Section 2.3 (L45) and Appendix D.1-D.3",
   "harmonisable": true,
   "harmonised_estimate": 0.17395330712343798,
   "harmonised_se": null,
   "poolable": false,
   "transformation": "percentage change: c = +0.19; theta = ln(1 + 0.19) = 0.173953 (equals the regression beta since S = exp(beta) - 1)",
   "reason_if_not": "Not poolable: no SE, CI or p-value is printed in the extract; the confidence interval appears only in a figure that is not reproduced, so SE(theta) cannot be obtained."
  },
  "S3": {
   "estimate_as_printed": "Treatment 44.29 min, Control 63.10 min (Overall 53.84); Improvement -29.79%***",
   "uncertainty_as_printed": "significance stars only (***; per the paper's scheme 0<p<=0.01); no SE or CI printed",
   "sample_as_printed": "181 subjects completed the task satisfactorily (one subject dropped)",
   "definition": "Percentage difference in mean task duration (minutes), Copilot (treatment) vs control without Copilot, holding accuracy constant (control duration predicted at treatment accuracy; Appendix B), randomised",
   "locator": "arxiv-2411.01067v2; Section 4.2, Table 3 (L51-L55), row 'Aggregate Results / All Content', columns Treatment, Control, Improvement",
   "harmonisable": true,
   "harmonised_estimate": -0.353679434958934,
   "harmonised_se": null,
   "poolable": false,
   "transformation": "percentage change: c = -0.2979; theta = ln(1 + (-0.2979)) = -0.353679 (ratio of displayed means ln(44.29/63.10) = -0.353962, sensitivity only)",
   "reason_if_not": "Not poolable: uncertainty is printed only as significance stars (p<=0.01); SE(theta) cannot be recovered from a threshold."
  },
  "S4": {
   "estimate_as_printed": "Aware 50.5, Control 56.9 (row labels: Time per Email (s)); % Time Savings 11.1",
   "uncertainty_as_printed": "p-value 0.471",
   "sample_as_printed": "Aware N=56, Control N=52 (of 167 analysts in three arms; 13 low-scoring responses filtered)",
   "definition": "Percentage time saving per processed email (Full Corpus, whole 25-email queue without resolve-benign protocol), Aware arm (agent verdicts, rationale, agent-ordered queue) vs Control (random order, no AI), randomised",
   "locator": "arxiv-2511.13860v1; Section 3.3, Table 3 (L86-L96), column 'Full Corpus', rows Aware/Control Time per Email, % Time Savings, p-value",
   "harmonisable": true,
   "harmonised_estimate": -0.11765804346823246,
   "harmonised_se": 0.1632205919720678,
   "poolable": true,
   "transformation": "percentage time saving 11.1% -> c = -0.111; theta = ln(1 + (-0.111)) = -0.117658; APPROXIMATE SE from two-sided p: SE = |theta|/z_(1-0.471/2) = 0.117658/0.720853 = 0.163221 (ratio of displayed means ln(50.5/56.9) = -0.119322, sensitivity only)",
   "reason_if_not": null
  },
  "S5": {
   "estimate_as_printed": "-0.278 (Treat x Post, Output, panel A, column (3))",
   "uncertainty_as_printed": "SE (0.295) (clustered at user level)",
   "sample_as_printed": "N = 18332 (user-day observations, users with affiliation)",
   "definition": "Difference-in-differences coefficient (Italy vs Austria/France, post vs pre ChatGPT ban) on an aggregate count of GitHub Output events (Push, Pull Requests, comments, Create, Issues)",
   "locator": "arxiv-2304.09339v1; Section 4, Table 1 (L80-L88), panel A 'Pre 27/28.03 - Post 03/04.04', row Treat x Post, column (3) Output (users with affiliation)",
   "harmonisable": false,
   "harmonised_estimate": null,
   "harmonised_se": null,
   "poolable": false,
   "transformation": "none: no transformation maps an effect on output counts onto ln(T_AI/T_noAI)",
   "reason_if_not": "Not harmonisable: the effect is on output counts (GitHub events) around the withdrawal of ChatGPT access in a non-randomised difference-in-differences design, not a comparison of task-completion times with and without AI on the same tasks."
  }
 },
 "k_extracted": 5,
 "k_harmonised": 4,
 "k_pooled": 2,
 "pooled_study_ids": [
  "S1",
  "S4"
 ],
 "fe_estimate": -0.18814332871148873,
 "fe_se": 0.10626483095451474,
 "fe_ci_low": -0.39642239738233764,
 "fe_ci_high": 0.02013573995936016,
 "re_estimate": -0.18814332871148873,
 "re_se": 0.10626483095451474,
 "re_ci_low": -0.39642239738233764,
 "re_ci_high": 0.02013573995936016,
 "q_stat": 0.3236860522786328,
 "q_df": 1,
 "i2": 0.0,
 "tau2": 0.0,
 "heterogeneity_interpretation": "low",
 "pooled_significant_5pct": false,
 "pooled_pct_change_re": -17.15040485348688,
 "pooled_pct_change_re_ci_low": -32.72775202989503,
 "pooled_pct_change_re_ci_high": 2.0339831514392115,
 "baseline_unweighted_mean_theta_harmonised": -0.13434604282593213,
 "baseline_unweighted_mean_theta_pooled": -0.17882902173411622,
 "sensitivity_ci_se_s1_re_estimate": -0.1891067723193802,
 "sensitivity_ci_se_s1_re_se": 0.10527305720065441,
 "sensitivity_ratio_s4_theta": -0.11932200485097105,
 "sensitivity_ratio_s4_re_estimate": -0.18967451746361488,
 "sensitivity_ratio_s4_re_se": 0.10689411567852646,
 "feasibility": "complete",
 "feasibility_reason": null,
 "data_issues": [
  {
   "issue": "S1 (Paradis): Table III states n = 96, while Section VI-B reports N=47 (AI) + N=46 (no-AI) = 93 completed tasks.",
   "handling": "Both sample statements recorded in sample_as_printed; coefficient and SE taken from Table III as designated; no effect on theta."
  },
  {
   "issue": "S1 (Paradis): Section VI-B says the AI group (Mean LogToT=4.6) was faster than the no-AI group (Mean LogToT=4.3), which contradicts the raw means (96 vs 114 min); the log means appear swapped.",
   "handling": "Not used; harmonisation relies on the Model 2 regression coefficient designated in studies.json."
  },
  {
   "issue": "S1 (Paradis): printed SE 0.14 and printed 95% CI [-0.51, 0.03] imply slightly different SEs (0.14 vs 0.1378); '21% faster' denotes a 21% reduction in time (1-exp(beta)).",
   "handling": "Printed SE used (convention for a coefficient with SE); CI-based SE reported as sensitivity_ci_se_s1_* keys."
  },
  {
   "issue": "S2 (Becker): the confidence interval for the 19% estimate is shown only in Figure 1/Figure 15, which are not reproduced in the extract.",
   "handling": "No SE imputed; S2 harmonised (theta = ln 1.19) but not poolable."
  },
  {
   "issue": "S3 (Bono & Xu): uncertainty only as significance stars with a non-standard scheme (* 0.05<p<=0.1, ** 0.01<p<=0.05, *** p<=0.01); displayed minutes give 44.29/63.10-1 = -29.81% vs printed -29.79%; control subjects could use other AI tools (25% did).",
   "handling": "Printed -29.79% used for theta; S3 not poolable; contamination noted as a limitation."
  },
  {
   "issue": "S4 (Bono): printed % Time Savings 11.1 does not equal 1-50.5/56.9 = 11.25% (Malicious and Non-Malicious columns also do not reproduce from displayed times); row labels say seconds per email, but Full Corpus values (50.5, 56.9) are inconsistent with the per-email Malicious/Non-Malicious values (2.8/1.9, 1.8/2.4), so the unit of the Full Corpus column is ambiguous.",
   "handling": "Printed % saving (11.1) used for theta (unit-free); ratio of displayed means used as sensitivity (sensitivity_ratio_s4_* keys); SE recovered approximately from p = 0.471."
  },
  {
   "issue": "S4 (Bono): the Aware arm differs from Control by both agent verdicts/rationales and agent-prioritised queue ordering; only a p-value is printed.",
   "handling": "Treated as AI vs no-AI; SE flagged as approximate."
  },
  {
   "issue": "S5 (Kreitmeir & Raschky): table notes print '** p<0.5' (presumably 0.05); row label rendered as '\\times TreatPost'; date line under the author block is a rendering artefact; whether Output is logged is ambiguous in Section 3.1.",
   "handling": "Values copied verbatim; S5 not harmonisable regardless of these issues."
  },
  {
   "issue": "Stray 'p<', 'p=' and footnote-number lines in the extracts.",
   "handling": "Ignored as rendering artefacts per data/README.md."
  }
 ],
 "n_input_rows": {
  "studies.json": 5,
  "README.md": 17,
  "sources/arxiv-2304.09339v1.txt": 108,
  "sources/arxiv-2410.12944v3.txt": 112,
  "sources/arxiv-2411.01067v2.txt": 109,
  "sources/arxiv-2507.09089v2.txt": 132,
  "sources/arxiv-2511.13860v1.txt": 162
 },
 "n_analysed": 5,
 "random_seed": null
}