1{2 "task_id": "T3-synthesis",3 "target_definition": "theta = ln(T_AI / T_noAI): natural log of the ratio of mean task-completion time with access to a generative-AI assistant to mean completion time without it, on the same tasks, from a randomised comparison; negative = faster with AI.",4 "items": {5 "S1": {6 "estimate_as_printed": "-0.24. (ExpCon coefficient, Model 2, LogToT; marker '.' = p<.1); text: beta_1=-0.24, 'about 21%' faster",7 "uncertainty_as_printed": "SE (0.14); 95% CI = [-0.51, 0.03]; p=0.086",8 "sample_as_printed": "n = 96 (Table III caption); Section VI-B: N=47 (AI), N=46 (no AI)",9 "definition": "OLS coefficient of the experimental-condition (AI) indicator on natural-log time on task, with developer-level covariates (Model 2); RCT, one complex enterprise coding task",10 "locator": "arxiv-2410.12944v3: Table III, row ExpCon, column (2) 'Dev' (+ SE line below); Section VI-C (CI and p for Model 2)",11 "harmonisable": true,12 "harmonised_estimate": -0.24,13 "harmonised_se": 0.14,14 "poolable": true,15 "transformation": "log-time regression coefficient: theta = b = -0.24; SE(theta) = SE(b) = 0.14 (printed CI gives (0.03-(-0.51))/(2*1.96) = 0.1378, consistent)",16 "reason_if_not": null17 },18 "S2": {19 "estimate_as_printed": "+19% ('allowing AI actually increases completion time by 19%'; 'implement issues in 19% more time on average')",20 "uncertainty_as_printed": "none printed in the extract (95% CI shown only in Figure 1, not reproduced)",21 "sample_as_printed": "16 developers, 246 tasks; 136 issues AI-allowed, 110 issues AI-disallowed",22 "definition": "S = E[time AI allowed]/E[time AI disallowed] - 1, estimated as exp(beta)-1 from OLS of log implementation time on AI indicator and log forecast time (Sections 2.3, D.1-D.3); issue-level RCT",23 "locator": "arxiv-2507.09089v2: Abstract and Section 3 'Results' opening paragraph (estimator Sections 2.3, D.1, D.3)",24 "harmonisable": true,25 "harmonised_estimate": 0.1739533071,26 "harmonised_se": null,27 "poolable": false,28 "transformation": "percentage change: theta = ln(1 + 0.19) = 0.173953",29 "reason_if_not": "Not poolable: no SE, CI or p-value for the headline estimate is printed in the extract; its CI appears only in Figure 1 (and Figure 15), which is not reproduced, so SE(theta) cannot be obtained."30 },31 "S3": {32 "estimate_as_printed": "Treatment 44.29 min, Control 63.10 min, Overall 53.84 min, Improvement -29.79%***",33 "uncertainty_as_printed": "significance stars only: *** (defined as 0<p<=0.01); no SE or CI printed",34 "sample_as_printed": "181 subjects (randomised half to treatment, half to control)",35 "definition": "percentage difference in mean task duration (minutes) for Copilot vs control, holding accuracy constant (control duration predicted at treatment accuracy, Appendix B); RCT",36 "locator": "arxiv-2411.01067v2: Table 3, block 'Aggregate Results', row 'All Content', columns Treatment/Control/Improvement",37 "harmonisable": true,38 "harmonised_estimate": -0.353679435,39 "harmonised_se": null,40 "poolable": false,41 "transformation": "percentage change: theta = ln(1 + (-0.2979)) = -0.353679 (ratio of printed durations ln(44.29/63.10) = -0.353962)",42 "reason_if_not": "Not poolable: the only uncertainty information is significance stars (***, p<=0.01); SE(theta) cannot be obtained."43 },44 "S4": {45 "estimate_as_printed": "Aware 50.5 s, Control 56.9 s per email; % Time Savings 11.1",46 "uncertainty_as_printed": "p-value 0.471",47 "sample_as_printed": "Aware N=56, Control N=52 (Blind N=59 not used); 167 analysts in total",48 "definition": "percentage time saving per email (full 25-email queue) of the Aware arm (agent verdicts, rationale, agent-prioritised queue) vs Control (no AI); RCT",49 "locator": "arxiv-2511.13860v1: Section 3.3, Table 3, column 'Full Corpus', rows Aware/Control time, % Time Savings, p-value",50 "harmonisable": true,51 "harmonised_estimate": -0.1176580435,52 "harmonised_se": 0.163220592,53 "poolable": true,54 "transformation": "percentage change c = -0.111 (11.1% saving): theta = ln(1 - 0.111) = -0.117658; APPROXIMATE SE from two-sided p: SE = |theta|/z(1-0.471/2) = 0.117658/0.720853 = 0.163221 (ratio of printed times ln(50.5/56.9) = -0.119322)",55 "reason_if_not": null56 },57 "S5": {58 "estimate_as_printed": "-0.278 (Treat x Post, Output, Panel A column (3))",59 "uncertainty_as_printed": "SE (0.295) (clustered at user level); not marked significant",60 "sample_as_printed": "N 18332",61 "definition": "difference-in-differences coefficient of Italy x post-ban on GitHub Output (count of push, pull requests, PR comments, commit comments, create, issues events; users with affiliation), Pre 27/28.03 vs Post 03/04.04 2023",62 "locator": "arxiv-2304.09339v1: Table 1, panel A, row Treat x Post (+ SE line), column (3) Output; row N",63 "harmonisable": false,64 "harmonised_estimate": null,65 "harmonised_se": null,66 "poolable": false,67 "transformation": "none (not harmonisable)",68 "reason_if_not": "Not harmonisable: the effect is on output counts (GitHub events) around the withdrawal of ChatGPT access in a non-randomised difference-in-differences design, not a comparison of task-completion times with vs without AI on the same tasks."69 }70 },71 "k_extracted": 5,72 "k_harmonised": 4,73 "k_pooled": 2,74 "fe_estimate": -0.1881433287,75 "fe_se": 0.106264831,76 "fe_ci_low": -0.3964223974,77 "fe_ci_high": 0.02013574,78 "re_estimate": -0.1881433287,79 "re_se": 0.106264831,80 "re_ci_low": -0.3964223974,81 "re_ci_high": 0.02013574,82 "q_stat": 0.3236860523,83 "i2": 0.0,84 "tau2": 0.0,85 "heterogeneity_interpretation": "low",86 "pooled_significant_5pct": false,87 "pooled_pct_change_re": -17.1504048535,88 "feasibility": "complete",89 "feasibility_reason": null,90 "data_issues": [91 {92 "issue": "S1: Section VI-B prints Mean LogToT=4.6 for the AI group and 4.3 for the no-AI group while stating the AI group was faster (96 vs 114 min); the two values appear swapped.",93 "handling": "Not used; the harmonised value is the Table III Model 2 coefficient, which agrees with the text of VI-C (CI, p, 21%)."94 },95 {96 "issue": "S1: Table III caption says n = 96 whereas Section VI-B reports N=47 + N=46 = 93.",97 "handling": "Both reported as printed in sample_as_printed; no effect on theta or SE."98 },99 {100 "issue": "S1: printed SE (0.14) and SE implied by the printed 95% CI [-0.51, 0.03] (0.1378) differ by rounding.",101 "handling": "Printed SE(b)=0.14 used (first convention); CI-based SE used as sensitivity analysis."102 },103 {104 "issue": "S2: the confidence interval of the headline +19% estimate is displayed only in Figure 1/15, which is not reproduced.",105 "handling": "harmonised_se = null, poolable = false; no remembered value substituted."106 },107 {108 "issue": "S3: printed improvement -29.79% differs slightly from the ratio of printed durations (44.29/63.10 - 1 = -29.81%).",109 "handling": "Printed percentage used (theta=-0.35368); duration-ratio value (-0.35396) reported in the transformation string."110 },111 {112 "issue": "S3: uncertainty only as significance stars with a non-standard key (*** 0<p<=0.01, * 0.05<p<=0.1).",113 "handling": "Harmonisable but not poolable per convention."114 },115 {116 "issue": "S4: printed % Time Savings 11.1 differs from the ratio of printed full-corpus times (50.5/56.9 -> 11.25%).",117 "handling": "Printed percentage and its p-value used; times-based theta used as sensitivity analysis."118 },119 {120 "issue": "S4: Table 3 Malicious/Non-Malicious columns (2.8, 1.8 vs 1.9, 2.4, labelled seconds) cannot be reconciled with Full Corpus times of 50.5/56.9 s; units of those columns ambiguous.",121 "handling": "Those columns are not used; only the Full Corpus column (a unit-free ratio) enters."122 },123 {124 "issue": "S4: SE recovered from a two-sided p-value (0.471) - approximate, assumes a Wald-type test on the log-ratio scale; the Aware arm also bundles agent queue prioritisation.",125 "handling": "Marked poolable with approximate SE, flagged in transformation and report."126 },127 {128 "issue": "S5: Table 1 notes print '** p<0.5' (presumably p<0.05); row label rendered as '\\times TreatPost'; header date 'August 24, 2026' is a rendering artefact; whether Output is logged is not stated unambiguously.",129 "handling": "Value reported as printed; S5 is not harmonisable for substantive reasons regardless."130 }131 ],132 "n_input_rows": {133 "studies.json": 5,134 "README.md": 17,135 "arxiv-2410.12944v3.txt": 112,136 "arxiv-2507.09089v2.txt": 132,137 "arxiv-2411.01067v2.txt": 109,138 "arxiv-2511.13860v1.txt": 162,139 "arxiv-2304.09339v1.txt": 108140 },141 "n_analysed": 5,142 "random_seed": null,143 "baseline_unweighted_mean_theta_harmonised": -0.1343460428,144 "baseline_unweighted_mean_theta_poolable": -0.1788290217,145 "pooled_studies": [146 "S1",147 "S4"148 ],149 "sensitivity_fe_with_S1_ci_se": -0.1891067723,150 "sensitivity_fe_with_S4_from_times": -0.1896745175,151 "fe_weights_pct": {152 "S1": 57.613338254,153 "S4": 42.386661746154 }155}