1{2 "task_id": "T3-synthesis",3 "target_definition": "theta = ln(T_AI / T_noAI): natural log of the ratio of mean task-completion time with access to a generative-AI assistant to mean completion time without it, for the same tasks, from a randomised comparison; negative values mean faster with AI.",4 "items": {5 "S1": {6 "estimate_as_printed": "ExpCon coefficient -0.24. (Model 2, logged time on task); text: 'effect of AI = 21%' faster, 'about 21%' in abstract",7 "uncertainty_as_printed": "SE (0.14); 95% CI = [-0.51, 0.03]; p=0.086 (NS); marker '.' p<.1",8 "sample_as_printed": "n = 96 (Table III); N=47 AI, N=46 no-AI (Sec VI-B)",9 "definition": "OLS coefficient of the AI-exposure indicator (ExpCon) on log time on task (LogToT), Model 2 with developer-level covariates; one complex enterprise coding task, randomised AI vs no-AI",10 "locator": "arxiv-2410.12944v3, Table III, row ExpCon (+ SE row below), column Model (2) 'Dev'; Sec VI-C sentence 'estimate of experimental condition on best-fit Model 2'",11 "harmonisable": true,12 "harmonised_estimate": -0.24,13 "harmonised_se": 0.14,14 "poolable": true,15 "transformation": "theta = b = -0.24 (log-time coefficient); SE(theta) = SE(b) = 0.14 (printed). Cross-check from CI: (0.03-(-0.51))/(2*1.96) = 0.1378",16 "reason_if_not": null17 },18 "S2": {19 "estimate_as_printed": "+19% (allowing AI increases completion time by 19%; 'implement issues in 19% more time on average')",20 "uncertainty_as_printed": "none printed (95% CI shown only in Figure 1 / Figure 15, not reproduced in the extract)",21 "sample_as_printed": "16 developers, 246 tasks (136 AI-allowed, 110 AI-disallowed issues)",22 "definition": "S = E[time | AI allowed]/E[time | AI disallowed] - 1 = exp(beta) - 1, beta from OLS of log implementation time on AI indicator and log forecast time; issue-level randomisation",23 "locator": "arxiv-2507.09089v2, Abstract and Section 3 opening paragraph ('19% more time'); estimator Sec 2.3 and App. D.1-D.3",24 "harmonisable": true,25 "harmonised_estimate": 0.173953,26 "harmonised_se": null,27 "poolable": false,28 "transformation": "theta = ln(1 + c) = ln(1 + 0.19) = 0.173953 (equivalently theta = beta, since S = exp(beta) - 1)",29 "reason_if_not": "Not poolable: no SE, CI or p-value for the 19% estimate is printed in the extract; the confidence interval appears only in Figure 1 (not reproduced), so SE(theta) cannot be obtained."30 },31 "S3": {32 "estimate_as_printed": "All Content: Overall 53.84, Treatment 44.29, Control 63.10 (minutes), Improvement -29.79%***",33 "uncertainty_as_printed": "significance stars only (*** = 0<p<=0.01 per the paper's definition); no SE or CI printed",34 "sample_as_printed": "181 subjects (treatment/control split ~half each; per-arm N not printed in extract)",35 "definition": "difference in task duration (minutes) between Copilot and control groups, holding accuracy constant (control duration predicted at treatment accuracy, App. B), all scenarios and tasks",36 "locator": "arxiv-2411.01067v2, Section 4.2, Table 3, block 'Aggregate Results', row 'All Content', columns Treatment / Control / Improvement",37 "harmonisable": true,38 "harmonised_estimate": -0.353679,39 "harmonised_se": null,40 "poolable": false,41 "transformation": "theta = ln(1 + c) = ln(1 - 0.2979) = -0.353679 (ratio of printed times: ln(44.29/63.10) = -0.353962)",42 "reason_if_not": "Not poolable: only significance stars (p<=0.01) are printed; no SE, CI or exact p-value, so SE(theta) cannot be obtained."43 },44 "S4": {45 "estimate_as_printed": "Full Corpus: Aware Time per Email (s) 50.5, Control Time per Email (s) 56.9, % Time Savings 11.1",46 "uncertainty_as_printed": "p-value 0.471",47 "sample_as_printed": "Aware N=56, Control N=52 (167 analysts across three arms after filtering)",48 "definition": "percentage time saving (Control vs Aware arm) in email-processing time over the full 25-email queue, without the resolve-benign protocol; Aware arm sees the agent's verdicts/rationales and an agent-ordered queue",49 "locator": "arxiv-2511.13860v1, Section 3.3, Table 3, column 'Full Corpus', rows Aware Time / Control Time / % Time Savings / p-value",50 "harmonisable": true,51 "harmonised_estimate": -0.117658,52 "harmonised_se": 0.163221,53 "poolable": true,54 "transformation": "time saving 11.1% => c = -0.111; theta = ln(1 - 0.111) = -0.117658; approximate SE = |theta| / z_(1-0.471/2) = 0.117658/0.720853 = 0.163221 (p assumed two-sided). APPROXIMATE SE.",55 "reason_if_not": null56 },57 "S5": {58 "estimate_as_printed": "Treat x Post -0.278 (Output, column 3, panel A)",59 "uncertainty_as_printed": "SE (0.295); no stars",60 "sample_as_printed": "N 18332 (user-day observations, users with affiliation)",61 "definition": "difference-in-differences coefficient of Italy x post-ban on the aggregate Output measure (count-based GitHub activity: pushes, PRs, comments, creates, issues), pre 27/28.03 vs post 03/04.04 2023, user FE",62 "locator": "arxiv-2304.09339v1, Section 4, Table 1, panel A, row 'Treat x Post' (+ SE row), column (3) Output, N row",63 "harmonisable": false,64 "harmonised_estimate": null,65 "harmonised_se": null,66 "poolable": false,67 "transformation": "none (not harmonisable)",68 "reason_if_not": "Not harmonisable: the effect is on output counts (GitHub events), not on task-completion time for the same tasks, and the contrast is a non-randomised difference-in-differences around the withdrawal of ChatGPT access (ban), not a randomised AI vs no-AI comparison."69 }70 },71 "k_extracted": 5,72 "k_harmonised": 4,73 "k_pooled": 2,74 "fe_estimate": -0.188143,75 "fe_se": 0.106265,76 "fe_ci_low": -0.396422,77 "fe_ci_high": 0.020136,78 "re_estimate": -0.188143,79 "re_se": 0.106265,80 "re_ci_low": -0.396422,81 "re_ci_high": 0.020136,82 "q_stat": 0.323686,83 "i2": 0.0,84 "tau2": 0.0,85 "heterogeneity_interpretation": "low",86 "pooled_significant_5pct": false,87 "pooled_pct_change_re": -17.150405,88 "feasibility": "complete",89 "feasibility_reason": null,90 "data_issues": [91 {92 "issue": "S1: Sec VI-B prints Mean LogToT = 4.6 for the AI group and 4.3 for the no-AI group, contradicting the minute means (96 vs 114) and the stated direction (AI faster).",93 "handling": "Not used; theta taken from the Table III Model 2 coefficient as instructed."94 },95 {96 "issue": "S1: Table III caption says n = 96 while Sec VI-B reports N=47 + N=46 = 93.",97 "handling": "Both reported verbatim in sample_as_printed; no effect on theta/SE."98 },99 {100 "issue": "S1: SE implied by the printed 95% CI [-0.51, 0.03] is 0.1378, vs printed SE 0.14 (rounding).",101 "handling": "Printed SE 0.14 used (first convention); CI-based SE reported in transformation as a cross-check."102 },103 {104 "issue": "S1: Table III model columns collapsed into one line per row (rendering); stray 'p<'/'p=' lines.",105 "handling": "Model 2 read as second value left to right per editorial note; consistent with -0.24 quoted in text. Stray lines ignored (README)."106 },107 {108 "issue": "S2: the 95% CI for the headline +19% is shown only in Figure 1/15, not reproduced; the raw ratio estimator gives 34% (footnote 12), a different estimator.",109 "handling": "Regression-based 19% used; SE set to null and study not pooled; 34% not used."110 },111 {112 "issue": "S3: printed improvement -29.79% differs slightly from the ratio of printed times (44.29/63.10 - 1 = -29.81%).",113 "handling": "Printed -29.79% used per the percentage-change convention; difference in theta < 0.0003."114 },115 {116 "issue": "S3: only significance stars printed, with unusual definitions (*** 0<p<=0.01, ** 0.01<p<=0.05, * 0.05<p<=0.1); 25% of control subjects used other AI tools.",117 "handling": "Not poolable; contamination noted as a limitation (estimate is incremental over other AI use)."118 },119 {120 "issue": "S4: Table 3 unit label '(s)' is implausible: per-email malicious/non-malicious values (2.8/1.8, 1.9/2.4) cannot average to the Full Corpus values (50.5/56.9); values look like per-email minutes and a full-queue total in minutes.",121 "handling": "theta is a unit-free log ratio, so unaffected; numbers reported as printed with the printed unit label."122 },123 {124 "issue": "S4: printed % time savings 11.1 vs 1 - 50.5/56.9 = 11.25%; p-value sidedness not stated.",125 "handling": "Printed 11.1% used; p treated as two-sided for the approximate SE."126 },127 {128 "issue": "S5: footnote star legend reads '** p<0.5' (apparent typo for 0.05); render-time date 'August 24, 2026'; text says April 11th excluded while panel B header lists 11.04.",129 "handling": "Irrelevant to the harmonisation (S5 not harmonisable); recorded only."130 }131 ],132 "n_input_rows": {133 "studies.json": 5,134 "README.md": 17,135 "arxiv-2410.12944v3.txt": 112,136 "arxiv-2507.09089v2.txt": 132,137 "arxiv-2411.01067v2.txt": 109,138 "arxiv-2511.13860v1.txt": 162,139 "arxiv-2304.09339v1.txt": 108140 },141 "n_analysed": 5,142 "random_seed": null,143 "baseline_unweighted_mean_theta_harmonised": -0.134346,144 "baseline_unweighted_mean_theta_poolable": -0.178829,145 "pooled_studies": [146 "S1",147 "S4"148 ],149 "q_df": 1,150 "q_pvalue": 0.569401,151 "s1_se_from_ci": 0.137755,152 "sensitivity_fe_s1_se_from_ci": [153 -0.189107,154 -0.395442,155 0.017228156 ],157 "sensitivity_fe_s4_theta_from_time_ratio": [158 -0.189675,159 -0.399187,160 0.019838161 ],162 "pooled_pct_change_re_ci": [163 -32.727752,164 2.033983165 ]166}