1{2 "task_id": "T3-synthesis",3 "target_definition": "theta = ln(T_AI / T_noAI): natural log of the ratio of mean task-completion time with generative-AI assistance to mean completion time without it, for the same tasks, from a randomised comparison (negative = faster with AI).",4 "items": {5 "S1": {6 "estimate_as_printed": "ExpCon coefficient, Model 2 (Dev): -0.24 (marked '.'); quoted in VI-C as beta_1=-0.24, i.e. AI effect (1-exp(beta_1))*100 = 21%",7 "uncertainty_as_printed": "SE (0.14); 95% CI = [-0.51, 0.03]; p=0.086 (NS)",8 "sample_as_printed": "n = 96 (Table III caption); VI-B: N=47 (AI), N=46 (no-AI)",9 "definition": "OLS coefficient on experimental condition (AI vs no-AI) with developer-level covariates; dependent variable = natural log of time on task (LogToT); randomised enterprise RCT, one complex coding task",10 "locator": "arxiv-2410.12944v3, Table III (L66), row ExpCon (L76) and SE row (L78), column (2) 'Dev' = Model 2; CI and p-value in Section VI-C (L58)",11 "harmonisable": true,12 "harmonised_estimate": -0.24,13 "harmonised_se": 0.14,14 "poolable": true,15 "transformation": "log-time coefficient: theta = b = -0.24; SE(theta) = SE(b) = 0.14 (CI-implied alternative (0.03-(-0.51))/(2*1.96) = 0.1378)",16 "reason_if_not": null17 },18 "S2": {19 "estimate_as_printed": "+19% (\"allowing AI actually increases completion time by 19%\"; \"they implement issues in 19% more time on average\")",20 "uncertainty_as_printed": "none printed in the extract (95% CI shown only in Figure 1/Figure 15, not reproduced)",21 "sample_as_printed": "16 developers, 246 tasks (136 issues AI-allowed, 110 AI-disallowed)",22 "definition": "S = E[time with AI allowed]/E[time with AI disallowed] - 1, estimated as exp(beta)-1 from OLS of log implementation time on AI indicator and log forecast time (issue-level randomisation)",23 "locator": "arxiv-2507.09089v2, Abstract (L34); Section 3 Results (L62); estimator Sections 2.3 and D.1-D.3",24 "harmonisable": true,25 "harmonised_estimate": 0.17395330712343798,26 "harmonised_se": null,27 "poolable": false,28 "transformation": "percentage change c = +0.19: theta = ln(1 + 0.19) = ln(1.19)",29 "reason_if_not": "Not poolable: no SE, CI or p-value for the 19% estimate is printed in the extract; its confidence interval appears only in figures that are not reproduced."30 },31 "S3": {32 "estimate_as_printed": "All Content: Overall 53.84 | Treatment 44.29 | Control 63.10 (minutes) | Improvement -29.79%***",33 "uncertainty_as_printed": "significance stars only: *** (0<p<=0.01); no SE or CI printed",34 "sample_as_printed": "181 subjects completed the task satisfactorily (one further subject dropped)",35 "definition": "Difference in mean task duration (minutes), Copilot vs control, holding accuracy constant (control duration predicted at the treatment group's accuracy, Appendix B); randomised assignment",36 "locator": "arxiv-2411.01067v2, Section 4.2, Table 3 (L87), 'Aggregate Results' block, row 'All Content' (L55), columns Treatment/Control/Improvement",37 "harmonisable": true,38 "harmonised_estimate": -0.353679434958934,39 "harmonised_se": null,40 "poolable": false,41 "transformation": "percentage change c = -0.2979: theta = ln(1 - 0.2979) = ln(0.7021) (ratio of printed durations 44.29/63.10 would give -0.35396)",42 "reason_if_not": "Not poolable: uncertainty is reported by significance stars only (***), so SE(theta) cannot be obtained."43 },44 "S4": {45 "estimate_as_printed": "Full Corpus: Aware Time per Email 50.5 s | Control Time per Email 56.9 s | % Time Savings 11.1",46 "uncertainty_as_printed": "p-value 0.471",47 "sample_as_printed": "Aware N=56, Control N=52 (Blind N=59; 167 analysts in total, 13 low-scoring responses filtered); 25-email queue per subject",48 "definition": "Mean time per email (seconds), Aware arm (agent verdicts, rationale and agent-ordered queue) vs Control arm (no AI), full queue; % time savings = reduction relative to Control; randomised three-arm RCT",49 "locator": "arxiv-2511.13860v1, Section 3.3, Table 3 (L88), rows Aware/Control Time per Email, % Time Savings, p-value (L90-L96), column 'Full Corpus'",50 "harmonisable": true,51 "harmonised_estimate": -0.11765804346823246,52 "harmonised_se": 0.1632205919720678,53 "poolable": true,54 "transformation": "time saving 11.1% -> c = -0.111: theta = ln(1 - 0.111) = ln(0.889) = -0.11766; approximate SE from two-sided p: SE = |theta| / z_(1-0.471/2) = 0.11766 / 0.72085 = 0.16322 (APPROXIMATE SE)",55 "reason_if_not": null56 },57 "S5": {58 "estimate_as_printed": "Treat x Post (Output, column 3, panel A, users with affiliation): -0.278",59 "uncertainty_as_printed": "SE (0.295), no significance stars",60 "sample_as_printed": "N = 18332 (user-day observations)",61 "definition": "Difference-in-differences coefficient on an aggregate GitHub Output count measure (Push, Pull Requests, PR comments, commit comments, Create, Issues) for Italian vs Austrian/French users, pre (27/28.03.2023) vs post (03/04.04.2023) the ChatGPT ban; user FE, user-clustered SE",62 "locator": "arxiv-2304.09339v1, Section 4, Table 1 (header L74), panel A, row Treat x Post (L80) and SE row (L82), column (3) Output; N row (L88)",63 "harmonisable": false,64 "harmonised_estimate": null,65 "harmonised_se": null,66 "poolable": false,67 "transformation": "none applied (not harmonisable)",68 "reason_if_not": "Not harmonisable: the printed effect is on output counts (activity volume) around a withdrawal of access in an observational DiD, not a comparison of task-completion times with and without AI on the same tasks."69 }70 },71 "k_extracted": 5,72 "k_harmonised": 4,73 "k_pooled": 2,74 "pooled_ids": [75 "S1",76 "S4"77 ],78 "fe_estimate": -0.18814332871148873,79 "fe_se": 0.10626483095451474,80 "fe_ci_low": -0.39642239738233764,81 "fe_ci_high": 0.02013573995936016,82 "re_estimate": -0.18814332871148873,83 "re_se": 0.10626483095451474,84 "re_ci_low": -0.39642239738233764,85 "re_ci_high": 0.02013573995936016,86 "q_stat": 0.3236860522786328,87 "q_df": 1,88 "q_pvalue": 0.5694008376973132,89 "i2": 0.0,90 "tau2": 0.0,91 "heterogeneity_interpretation": "low",92 "pooled_significant_5pct": false,93 "pooled_pct_change_re": -17.15040485348688,94 "pooled_pct_change_re_ci": [95 -32.72775202989503,96 2.033983151439211597 ],98 "weights_fe": {99 "S1": 0.5761333825403874,100 "S4": 0.4238666174596126101 },102 "weights_re": {103 "S1": 0.5761333825403874,104 "S4": 0.4238666174596126105 },106 "baseline_unweighted_mean_theta_harmonised": -0.13434604282593213,107 "baseline_unweighted_mean_theta_poolable": -0.17882902173411622,108 "baseline_vote_count_faster_with_ai": 3,109 "sens_re_estimate_s1_ci_se": -0.1891067723193802,110 "sens_re_se_s1_ci_se": 0.10527305720065441,111 "sens_re_ci_low_s1_ci_se": -0.39544196443266283,112 "sens_re_ci_high_s1_ci_se": 0.01722841979390244,113 "feasibility": "complete",114 "feasibility_reason": null,115 "data_issues": [116 {117 "issue": "S1: Section VI-B (L48) prints Mean LogToT = 4.6 for the AI group and 4.3 for the no-AI group while stating the AI group was faster (96 vs 114 min; ln 96 = 4.56, ln 114 = 4.74): internally inconsistent.",118 "handling": "Mean LogToT values not used; theta taken from the Table III Model 2 coefficient."119 },120 {121 "issue": "S1: Section VI-B reports N=47+46=93 developers while Table III states n = 96.",122 "handling": "Both printed as is in sample_as_printed; not needed for theta or SE."123 },124 {125 "issue": "S1: the CI [-0.51, 0.03] implies SE = 0.1378 whereas Table III prints SE = 0.14 (rounding).",126 "handling": "Printed SE(b) = 0.14 used (first harmonisation convention; reproduces p = 0.086 under a normal approximation); sensitivity with the CI-implied SE reported as sens_* keys."127 },128 {129 "issue": "S3: printed durations give 44.29/63.10 - 1 = -29.81% whereas the printed improvement is -29.79%.",130 "handling": "Printed improvement c = -0.2979 used (theta = ln(1+c)); the ratio-based value differs by < 0.001 on the theta scale; S3 is not pooled anyway."131 },132 {133 "issue": "S4: printed Full Corpus times give 1 - 50.5/56.9 = 11.25% savings, not the printed 11.1%; the Malicious/Non-Malicious columns (2.8, 1.8, 1.9, 2.4, labelled seconds) do not match the full-corpus magnitude (50.5/56.9 s) and their implied savings differ from the printed -52.9/23.6 (ambiguous units/rounding).",134 "handling": "Only the Full Corpus column is used, with the printed % time savings (11.1) and p-value; the ambiguous columns are not used."135 },136 {137 "issue": "S4: only a p-value is printed; the SE is recovered as |theta|/z_(1-p/2), which assumes a normal test statistic on the log-ratio scale (the paper's test may be on another scale).",138 "handling": "S4 pooled with an explicitly approximate SE, as allowed by the task conventions; flagged in report."139 },140 {141 "issue": "S5: Table 1 notes print the significance legend as '** p<0.5' (L108), presumably a typo for p<0.05.",142 "handling": "Irrelevant to the analysis (S5 not harmonisable); recorded."143 }144 ],145 "n_input_rows": {146 "studies.json": 5,147 "arxiv-2410.12944v3.txt": 112,148 "arxiv-2507.09089v2.txt": 132,149 "arxiv-2411.01067v2.txt": 109,150 "arxiv-2511.13860v1.txt": 162,151 "arxiv-2304.09339v1.txt": 108152 },153 "n_analysed": 5,154 "random_seed": null155}