1{2 "task_id": "T3-synthesis",3 "target_definition": "theta = ln(T_AI / T_noAI): natural log of the ratio of mean task-completion time with access to the generative-AI assistant to mean completion time without it, on the same tasks, from a randomised comparison; negative = faster with AI.",4 "items": {5 "S1": {6 "estimate_as_printed": "ExpCon coefficient (Model 2) = -0.24. (marker '.' = p<.1) on LogToT; text: beta_1=-0.24, effect of AI = 21% (faster)",7 "uncertainty_as_printed": "SE (0.14); 95% CI = [-0.51, 0.03]; p=0.086 (NS)",8 "sample_as_printed": "n = 96 (Table III); time-on-task means N=47 (AI) and N=46 (no AI) in Sec. VI-B",9 "definition": "OLS coefficient of the AI experimental-condition indicator on the natural log of time on task, controlling for developer-level covariates (Model 2); randomised, one common enterprise coding task",10 "locator": "arxiv-2410.12944v3, Table III, row ExpCon (+ SE row below), column (2) 'Dev' [extract lines 76-78]; Section VI-C, 95% CI and p-value [line 58]",11 "harmonisable": true,12 "harmonised_estimate": -0.24,13 "harmonised_se": 0.14,14 "poolable": true,15 "transformation": "theta = b = -0.24; SE(theta) = SE(b) = 0.14 (cross-check from printed CI: (0.03 - (-0.51))/(2*1.96) = 0.1378)",16 "reason_if_not": null17 },18 "S2": {19 "estimate_as_printed": "+19% (allowing AI increases completion time by 19%; 'implement issues in 19% more time on average')",20 "uncertainty_as_printed": "none printed (the 95% CI is shown only in Figure 1, which is not reproduced in the extract)",21 "sample_as_printed": "16 developers; 246 tasks (136 issues AI-allowed, 110 AI-disallowed)",22 "definition": "S = E[time with AI allowed]/E[time with AI disallowed] - 1, estimated as exp(beta)-1 from a log-linear OLS of total implementation time on the AI indicator and log forecast time (issue-level randomisation)",23 "locator": "arxiv-2507.09089v2, Abstract [line 34] and Section 3 Results opening [line 62]; estimator Sections 2.3 and D.1-D.3",24 "harmonisable": true,25 "harmonised_estimate": 0.17395330712343798,26 "harmonised_se": null,27 "poolable": false,28 "transformation": "theta = ln(1 + c) = ln(1 + 0.19) = 0.173953",29 "reason_if_not": "Not poolable: no SE, CI or p-value for the 19% estimate is printed in the extract; the confidence interval appears only in Figure 1, which is not reproduced, so SE(theta) cannot be obtained."30 },31 "S3": {32 "estimate_as_printed": "All Content: Overall 53.84 min, Treatment 44.29 min, Control 63.10 min, Improvement -29.79%***",33 "uncertainty_as_printed": "significance stars only: *** (defined as 0<p<=0.01); no SE or CI printed",34 "sample_as_printed": "181 subjects completed the task satisfactorily (randomised half to Copilot)",35 "definition": "Percentage difference in mean task duration (minutes), Copilot vs control, holding accuracy constant (control duration predicted at treatment accuracy; Appendix B), all tasks across three IT-admin scenarios",36 "locator": "arxiv-2411.01067v2, Section 4.2, Table 3, row 'Aggregate Results - All Content', columns Treatment/Control/Improvement [line 55]",37 "harmonisable": true,38 "harmonised_estimate": -0.353679434958934,39 "harmonised_se": null,40 "poolable": false,41 "transformation": "theta = ln(1 + c) = ln(1 + (-0.2979)) = -0.353679 (printed ratio check: ln(44.29/63.1) = -0.353962)",42 "reason_if_not": "Not poolable: uncertainty is reported only as significance stars (*** = p<=0.01); bootstrap SEs/CIs are not printed, so SE(theta) cannot be obtained."43 },44 "S4": {45 "estimate_as_printed": "Full Corpus: Aware Time per Email (s) 50.5, Control Time per Email (s) 56.9, % Time Savings 11.1",46 "uncertainty_as_printed": "p-value 0.471",47 "sample_as_printed": "Control N=52, Aware N=56 (Blind N=59; 167 analysts in total; 13 responses scoring <40% filtered: 3 Aware, 6 Blind, 4 Control)",48 "definition": "Percentage time saving per processed email (mean time per email), Aware arm (agent queue ordering + verdicts/rationales) vs Control (no AI), full 25-email queue",49 "locator": "arxiv-2511.13860v1, Section 3.3, Table 3, column 'Full Corpus', rows Aware/Control time, % Time Savings, p-value [lines 88-96]",50 "harmonisable": true,51 "harmonised_estimate": -0.11765804346823246,52 "harmonised_se": 0.1632205919720678,53 "poolable": true,54 "transformation": "c = -11.1/100 = -0.111; theta = ln(1 + c) = ln(0.889) = -0.117658; approximate SE = |theta| / z_(1-p/2) = 0.117658 / z_(1-0.471/2) = 0.117658 / 0.720853 = 0.163221 (approximate SE from p-value only)",55 "reason_if_not": null56 },57 "S5": {58 "estimate_as_printed": "Treat x Post (Output, column 3, panel A) = -0.278",59 "uncertainty_as_printed": "SE (0.295) (clustered at user level); not significant",60 "sample_as_printed": "N = 18332 (user-day observations, users with affiliation)",61 "definition": "Difference-in-differences coefficient on an aggregate GitHub output count measure (Push, PRs, comments, Create, Issues) for Italian vs Austrian/French users around Italy's ChatGPT ban (pre 27/28.03 vs post 03/04.04.2023)",62 "locator": "arxiv-2304.09339v1, Section 4, Table 1, panel A, row Treat x Post (+ SE row), column (3) Output; N row [lines 80-88]",63 "harmonisable": false,64 "harmonised_estimate": null,65 "harmonised_se": null,66 "poolable": false,67 "transformation": "none (not harmonisable)",68 "reason_if_not": "Not harmonisable: the effect is on output counts (activity volume) around the withdrawal of access, not a comparison of completion times with vs without AI on the same tasks, and it is not a randomised comparison."69 }70 },71 "k_extracted": 5,72 "k_harmonised": 4,73 "k_pooled": 2,74 "pooled_studies": [75 "S1",76 "S4"77 ],78 "fe_estimate": -0.18814332871148873,79 "fe_se": 0.10626483095451474,80 "fe_ci_low": -0.39642239738233764,81 "fe_ci_high": 0.02013573995936016,82 "re_estimate": -0.18814332871148873,83 "re_se": 0.10626483095451474,84 "re_ci_low": -0.39642239738233764,85 "re_ci_high": 0.02013573995936016,86 "q_stat": 0.3236860522786328,87 "q_df": 1,88 "q_pvalue": 0.5694008376973132,89 "i2": 0.0,90 "tau2": 0.0,91 "heterogeneity_interpretation": "low",92 "pooled_significant_5pct": false,93 "pooled_pct_change_re": -17.15040485348688,94 "baseline_unweighted_mean_poolable": -0.17882902173411622,95 "baseline_unweighted_mean_harmonised": -0.13434604282593213,96 "sensitivity_s4_ratio_of_means": {97 "theta_S4": -0.11932200485097105,98 "se_S4": 0.16552891492138308,99 "fe_estimate": -0.18967451746361488100 },101 "sensitivity_s1_se_from_ci": {102 "se_S1": 0.13775510204081634,103 "fe_estimate": -0.1891067723193802104 },105 "feasibility": "complete",106 "feasibility_reason": null,107 "data_issues": [108 {109 "issue": "S1 (Paradis): Sec. VI-B prints Mean LogToT = 4.6 for the AI group and 4.3 for the no-AI group, which contradicts the reported faster AI group (96 vs 114 min).",110 "handling": "Not used; the extracted quantity is the Table III Model 2 coefficient, which is internally consistent with the Sec. VI-C CI and p-value."111 },112 {113 "issue": "S1: Table III states n = 96 while Sec. VI-B reports N = 47 + 46 = 93 for time on task.",114 "handling": "Both reported verbatim in sample_as_printed; does not affect theta or SE."115 },116 {117 "issue": "S1: printed SE 0.14 vs SE implied by the printed 95% CI [-0.51, 0.03] = 0.1378 (rounding).",118 "handling": "Printed SE(b)=0.14 used per convention; CI-implied SE recorded in transformation as cross-check."119 },120 {121 "issue": "S1: Table III rendering collapses four model columns into one line with the variable name at the end and garbles the 'Adj. R^2' / 'p value' row labels.",122 "handling": "Columns read left to right as Models 1-4 per the editorial note; Model 2 = -0.24 confirmed by text (Sec. VI-C)."123 },124 {125 "issue": "S2 (Becker): the confidence interval of the headline 19% estimate is only displayed in Figure 1, not printed in the extract.",126 "handling": "Harmonised (theta = ln 1.19) but not poolable; no remembered value substituted."127 },128 {129 "issue": "S2: a raw ratio estimate of 34% is also printed (footnote 12), differing from the 19% regression estimate.",130 "handling": "The regression-based 19% headline estimate requested in studies.json is used."131 },132 {133 "issue": "S3 (Bono & Xu): printed Improvement -29.79% vs ratio of printed means 44.29/63.10 - 1 = -29.81%.",134 "handling": "Printed percentage used (difference 0.0003 on theta scale, rounding)."135 },136 {137 "issue": "S3: only significance stars are printed, with a nonstandard definition (* 0.05<p<=0.1, ** 0.01<p<=0.05, *** p<=0.01).",138 "handling": "SE not recoverable; study marked not poolable."139 },140 {141 "issue": "S4 (Bono 2025): printed % Time Savings 11.1 vs 1 - 50.5/56.9 = 11.25% from printed means.",142 "handling": "Printed percentage and p-value used; sensitivity with the ratio of means reported in results (sensitivity_s4_ratio_of_means)."143 },144 {145 "issue": "S4: Table 3 Malicious/Non-Malicious 'time per email (s)' values (1.8-2.8) are implausibly small vs the Full Corpus (~50 s), and the Malicious % (-52.9) does not match 2.8/1.9; units ambiguous in those columns.",146 "handling": "Only the Full Corpus column is used; anomaly documented."147 },148 {149 "issue": "S4: arm sizes (52/56/59, summing to 167) appear to be before the filtering of 13 responses; analysed N per arm not printed.",150 "handling": "Reported verbatim in sample_as_printed; not needed for theta or the p-value-based SE."151 },152 {153 "issue": "S4: SE is approximate, recovered from a two-sided p-value (0.471) assuming normality.",154 "handling": "Marked as approximate; S4 enters pooling per the stated convention."155 },156 {157 "issue": "S5 (Kreitmeir & Raschky): table notes print '** p<0.5' (presumably p<0.05); row label rendered as '\\times TreatPost'; date line 'August 24, 2026' is a rendering artefact.",158 "handling": "No effect: S5 is not harmonisable."159 }160 ],161 "n_input_rows": {162 "README.md": 17,163 "studies.json": 5,164 "arxiv-2304.09339v1.txt": 108,165 "arxiv-2410.12944v3.txt": 112,166 "arxiv-2411.01067v2.txt": 109,167 "arxiv-2507.09089v2.txt": 132,168 "arxiv-2511.13860v1.txt": 162169 },170 "n_analysed": 5,171 "random_seed": null172}