1{2 "task_id": "T3-synthesis",3 "target_definition": "theta = ln(T_AI / T_noAI): natural log of the ratio of mean task-completion time with generative-AI assistance to mean completion time without it, on the same tasks, from a randomised comparison (negative = faster with AI).",4 "items": {5 "S1": {6 "estimate_as_printed": "ExpCon coefficient (Model 2, LogToT) = -0.24 (marked '.', p<.1); text: effect of AI = 21% faster",7 "uncertainty_as_printed": "SE (0.14); 95% CI = [-0.51, 0.03]; p=0.086",8 "sample_as_printed": "n = 96 (Table III caption); Section VI-B: N=47 AI, N=46 no-AI",9 "definition": "OLS coefficient on experimental condition (AI vs no-AI exposure) with natural-log time on task as dependent variable, controlling for developer-level factors (AvgProgHrsDay, Level); one complex enterprise coding task, randomised",10 "locator": "arxiv-2410.12944v3, Table III, row 'ExpCon', column (2) 'Dev' (value -0.24., SE line (0.14)); CI and p-value in Section VI-C text",11 "harmonisable": true,12 "harmonised_estimate": -0.24,13 "harmonised_se": 0.14,14 "poolable": true,15 "transformation": "theta = b = -0.24; SE(theta) = SE(b) = 0.14 (printed). Cross-check from printed CI: (0.03-(-0.51))/(2*1.96) = 0.1378",16 "reason_if_not": null17 },18 "S2": {19 "estimate_as_printed": "+19% (\"allowing AI actually increases completion time by 19%\"; \"implement issues in 19% more time on average\")",20 "uncertainty_as_printed": "none printed in extract (95% CI shown only in Figure 1, not reproduced)",21 "sample_as_printed": "16 developers, 246 tasks (136 issues AI-allowed, 110 AI-disallowed)",22 "definition": "S = E[completion time with AI allowed]/E[completion time with AI disallowed] - 1, estimated as exp(beta)-1 from log-linear OLS of total implementation time on AI indicator and log forecast time; issue-level randomisation",23 "locator": "arxiv-2507.09089v2, Abstract and Section 3 'Results' opening paragraph (19%); estimator in Section 2.3 and Appendix D.1-D.3",24 "harmonisable": true,25 "harmonised_estimate": 0.17395330712343798,26 "harmonised_se": null,27 "poolable": false,28 "transformation": "theta = ln(1 + c) = ln(1 + 0.19) = 0.1740 (equivalently theta = beta-hat, since S = exp(beta)-1)",29 "reason_if_not": "No SE, CI or p-value for the 19% estimate is printed in the extract; its confidence interval appears only in Figure 1, which is not reproduced. SE cannot be obtained, so not poolable."30 },31 "S3": {32 "estimate_as_printed": "-29.79%*** (Treatment 44.29 min, Control 63.10 min; Overall 53.84 min)",33 "uncertainty_as_printed": "significance stars only: *** (0<p<=0.01)",34 "sample_as_printed": "181 subjects completed the task satisfactorily (treatment/control split not printed in extract)",35 "definition": "Percentage difference in mean task duration (minutes), Copilot vs control, holding accuracy constant (control duration predicted at treatment accuracy via a control-group regression of duration on accuracy); all content, randomised",36 "locator": "arxiv-2411.01067v2, Table 3, block 'Aggregate Results', row 'All Content', columns Treatment / Control / Improvement",37 "harmonisable": true,38 "harmonised_estimate": -0.353679434958934,39 "harmonised_se": null,40 "poolable": false,41 "transformation": "theta = ln(1 + c) = ln(1 - 0.2979) = -0.3537 (ratio of printed means: ln(44.29/63.10) = -0.3540)",42 "reason_if_not": "Only significance stars (***, p<=0.01) are printed; no SE, CI or exact p-value, so SE cannot be obtained and the study is not poolable."43 },44 "S4": {45 "estimate_as_printed": "% Time Savings 11.1 (Aware 50.5 s per email, Control 56.9 s per email), Full Corpus",46 "uncertainty_as_printed": "p-value 0.471",47 "sample_as_printed": "Aware N=56, Control N=52 (167 analysts across three arms; 25-email queues)",48 "definition": "Percentage time saving in mean time per email (Aware arm: agent verdicts/rationale and agent-ordered queue vs Control: random order, no AI), across the full queue; randomised",49 "locator": "arxiv-2511.13860v1, Section 3.3, Table 3, column 'Full Corpus', rows 'Aware Time per Email (s)', 'Control Time per Email (s)', '% Time Savings', 'p-value'",50 "harmonisable": true,51 "harmonised_estimate": -0.11765804346823246,52 "harmonised_se": 0.1632205919720678,53 "poolable": true,54 "transformation": "c = -0.111 (11.1% saving); theta = ln(1 - 0.111) = -0.1177; approximate SE = |theta|/z_(1-0.471/2) = 0.1177/0.7209 = 0.1632 (APPROXIMATE SE recovered from p-value). Ratio of printed means: ln(50.5/56.9) = -0.1193",55 "reason_if_not": null56 },57 "S5": {58 "estimate_as_printed": "Treat x Post = -0.278 (Output, column 3, panel A, users with affiliation)",59 "uncertainty_as_printed": "SE (0.295); no stars",60 "sample_as_printed": "N = 18332",61 "definition": "Difference-in-differences coefficient on the aggregate GitHub Output measure (sum of Push, Pull Requests, PR comments, commit comments, Create, Issues events; the text describes log counts for these measures) for Italian vs Austrian/French users, first two business days after vs before the ChatGPT ban",62 "locator": "arxiv-2304.09339v1, Table 1, panel A (Pre 27/28.03 - Post 03/04.04), row 'Treat x Post', column (3) 'Output'; N from row 'N'",63 "harmonisable": false,64 "harmonised_estimate": null,65 "harmonised_se": null,66 "poolable": false,67 "transformation": "none applied (not harmonisable)",68 "reason_if_not": "The printed effect is on output counts (GitHub events), not task-completion time on the same tasks, and it comes from a natural experiment (withdrawal of access via a national ban, DiD) rather than a randomised AI-vs-no-AI comparison of completion times. It cannot be mapped onto theta = ln(T_AI/T_noAI)."69 }70 },71 "k_extracted": 5,72 "k_harmonised": 4,73 "k_pooled": 2,74 "fe_estimate": -0.18814332871148873,75 "fe_se": 0.10626483095451474,76 "fe_ci_low": -0.39642239738233764,77 "fe_ci_high": 0.02013573995936016,78 "re_estimate": -0.18814332871148873,79 "re_se": 0.10626483095451474,80 "re_ci_low": -0.39642239738233764,81 "re_ci_high": 0.02013573995936016,82 "q_stat": 0.3236860522786328,83 "i2": 0.0,84 "tau2": 0.0,85 "heterogeneity_interpretation": "low",86 "pooled_significant_5pct": false,87 "pooled_pct_change_re": -17.15040485348688,88 "feasibility": "complete",89 "feasibility_reason": null,90 "data_issues": [91 {92 "issue": "S1 (arxiv-2410.12944v3, Section VI-B): the AI group is stated to be faster, yet its printed Mean LogToT (4.6) is larger than the no-AI group's (4.3); neither matches the printed raw means (exp(4.6)~99, exp(4.3)~74 vs 96 and 114 min). Internal inconsistency of the source.",93 "handling": "Not used; the harmonised value comes from the Table III Model 2 coefficient as instructed by studies.json."94 },95 {96 "issue": "S1: Table III caption says n = 96 while Section VI-B reports N=47 + N=46 = 93 for the raw means.",97 "handling": "Both sample sizes reported as printed; no effect on theta/SE, which are taken directly from Table III."98 },99 {100 "issue": "S1: Table III rendering collapses the four model columns into one cell per row, and the R^2 / p-value rows have garbled labels ('R^{2} ... Adj.', '... p value').",101 "handling": "Column order Models 1-4 left to right per the editorial note; Model 2 ExpCon = -0.24 (0.14) cross-checked against the text (beta_1 = -0.24, CI [-0.51, 0.03], p = 0.086)."102 },103 {104 "issue": "S1: two possible SEs: printed SE 0.14 (rounded to 2 dp) and the SE implied by the printed 95% CI, (0.03+0.51)/3.92 = 0.1378.",105 "handling": "Primary analysis uses the printed SE 0.14; sensitivity with the CI-based SE reported (FE = -0.1891)."106 },107 {108 "issue": "S2: the 95% CI of the headline 19% estimate is displayed only in Figure 1 (not reproduced); the raw ratio of means (34%) is also printed but is not the regression estimate.",109 "handling": "Harmonised the regression-based 19% (theta = ln 1.19); SE set to null and study not pooled; 34% not used."110 },111 {112 "issue": "S3: the printed improvement -29.79% differs slightly from the ratio of the printed means (44.29/63.10 - 1 = -29.81%), presumably due to rounding of the means; sign conventions differ between text ('29.79% reduction') and table ('-29.79%').",113 "handling": "Used the printed improvement c = -0.2979 per the harmonisation convention; difference in theta < 0.001."114 },115 {116 "issue": "S4: Table 3 header says 'Time per Email (s)', but the Malicious/Non-Malicious columns (2.8, 1.8, 1.9, 2.4) are implausible as seconds next to Full Corpus 50.5/56.9 s and do not reproduce their printed % savings (e.g. 1-2.8/1.9 = -47.4% vs printed -52.9); likely minutes and/or rounding. Also 1-50.5/56.9 = 11.25% vs printed 11.1.",117 "handling": "Only the Full Corpus column is used, with the printed % Time Savings (11.1) as the harmonisation input; sign convention: positive savings means less time with AI, so c = -0.111."118 },119 {120 "issue": "S4: only a two-sided p-value (0.471) is printed for the full-corpus time saving; the test underlying it (likely on raw seconds, not logs) is not described in the extract.",121 "handling": "Approximate SE = |theta|/z_(1-p/2) as permitted; study pooled with an approximate SE and flagged."122 },123 {124 "issue": "S5: Table 1 notes print the significance key as '** p<0.5' (presumably p<0.05); the row label renders as '\\times TreatPost'.",125 "handling": "Recorded as printed; S5 is not harmonisable regardless."126 },127 {128 "issue": "Stray 'p<' / 'p=' lines and footnote numbers in the extracts (rendering artefacts per data/README.md).",129 "handling": "Ignored."130 }131 ],132 "n_input_rows": {133 "studies.json": 5,134 "README.md": 17,135 "arxiv-2304.09339v1.txt": 108,136 "arxiv-2410.12944v3.txt": 112,137 "arxiv-2411.01067v2.txt": 109,138 "arxiv-2507.09089v2.txt": 132,139 "arxiv-2511.13860v1.txt": 162140 },141 "n_analysed": 5,142 "random_seed": null143}