1{2 "task_id": "T3-synthesis",3 "target_definition": "theta = ln(T_AI / T_noAI): natural log of the ratio of mean task-completion time with generative-AI assistance to mean completion time without it, for the same tasks, from a randomised comparison (negative = faster with AI).",4 "items": {5 "S1": {6 "estimate_as_printed": "-0.24. (beta_1 for ExpCon, Model 2, Table III; Section VI-C: beta_1=-0.24, i.e. 21% faster)",7 "uncertainty_as_printed": "SE (0.14); 95% CI = [-0.51, 0.03], p=0.086 (NS)",8 "sample_as_printed": "n = 96 (Table III caption); Section VI-B: N=47 AI, N=46 no-AI",9 "definition": "OLS coefficient on experimental condition (AI exposure) with natural-log time on task (LogToT) as dependent variable, developer-level covariates (Model 2), one enterprise coding task, randomised",10 "locator": "arxiv-2410.12944v3, Section VI-C Table III, row ExpCon, column (2) 'Dev' (second value on collapsed line L76; SE line L78); CI and p in Section VI-C text",11 "harmonisable": true,12 "harmonised_estimate": -0.24,13 "harmonised_se": 0.14,14 "poolable": true,15 "transformation": "log-time coefficient: theta = b = -0.24; SE(theta) = SE(b) = 0.14 (printed). Check: SE from CI = (0.03-(-0.51))/(2*1.96) = 0.1378",16 "reason_if_not": null17 },18 "S2": {19 "estimate_as_printed": "+19% (allowing AI increases completion time by 19%; regression-based, Section 3 / Section D.1)",20 "uncertainty_as_printed": "none printed (95% CI shown only in Figure 1 / Figure 15, not reproduced in the extract)",21 "sample_as_printed": "16 developers, 246 tasks; 136 issues AI-allowed, 110 AI-disallowed",22 "definition": "S = E[time AI allowed]/E[time AI disallowed] - 1, estimated as exp(beta)-1 from OLS of log implementation time on AI indicator and log forecast time; issue-level randomisation",23 "locator": "arxiv-2507.09089v2, Abstract and Section 3 'Results' (opening paragraph); estimator Sections 2.3, D.1-D.3",24 "harmonisable": true,25 "harmonised_estimate": 0.17395330712343798,26 "harmonised_se": null,27 "poolable": false,28 "transformation": "percentage change in time: theta = ln(1 + c) = ln(1 + 0.19) = 0.173953",29 "reason_if_not": "No SE, CI or p-value printed in the extract; the CI exists only in a figure that is not reproduced, so SE(theta) cannot be obtained (harmonisable, not poolable)."30 },31 "S3": {32 "estimate_as_printed": "Overall 53.84, Treatment 44.29, Control 63.10 (minutes); Improvement -29.79%***",33 "uncertainty_as_printed": "significance stars only: *** (paper's definition 0<p<=0.01); no SE or CI printed",34 "sample_as_printed": "181 subjects (randomised half treatment, half control)",35 "definition": "difference in mean task duration (minutes) between Copilot and control groups across all tasks, holding accuracy constant (control duration predicted at treatment accuracy, Appendix B), randomised",36 "locator": "arxiv-2411.01067v2, Section 4.2 Table 3, 'Aggregate Results' block, row 'All Content', columns Treatment/Control/Improvement",37 "harmonisable": true,38 "harmonised_estimate": -0.353679434958934,39 "harmonised_se": null,40 "poolable": false,41 "transformation": "percentage change in time: theta = ln(1 + c) = ln(1 + (-0.2979)) = -0.353679 (cf. ln(44.29/63.10) = -0.353962)",42 "reason_if_not": "Only significance stars (p<=0.01) are printed; a p-value threshold does not identify SE(theta) (harmonisable, not poolable)."43 },44 "S4": {45 "estimate_as_printed": "Aware 50.5 s, Control 56.9 s per email; % Time Savings 11.1",46 "uncertainty_as_printed": "p-value 0.471",47 "sample_as_printed": "Aware N=56, Control N=52 (167 analysts across three arms incl. Blind N=59)",48 "definition": "percentage time saving in mean time per email over the full 25-email queue, Aware (agent verdicts/rationale + prioritised queue) vs Control (no AI), randomised arms",49 "locator": "arxiv-2511.13860v1, Section 3.3 Table 3 'Comparison of Email Processing Times', column 'Full Corpus', rows Aware/Control time, % Time Savings, p-value",50 "harmonisable": true,51 "harmonised_estimate": -0.11765804346823246,52 "harmonised_se": 0.1632205919720678,53 "poolable": true,54 "transformation": "percentage change in time c = -11.1/100 = -0.111; theta = ln(1 + c) = -0.117658; approximate SE from two-sided p: SE = |theta|/z(1-0.471/2) = 0.117658/0.720853 = 0.163221 (approximate SE)",55 "reason_if_not": null56 },57 "S5": {58 "estimate_as_printed": "-0.278 (Treat x Post, Output, column 3, panel A)",59 "uncertainty_as_printed": "SE (0.295) (clustered at user level); no significance star",60 "sample_as_printed": "N = 18332 (user-day observations, users with affiliation)",61 "definition": "difference-in-differences coefficient on aggregate GitHub Output measure (sum of Push, PR, PR comments, commit comments, Create, Issues events) for Italian vs Austrian/French users, pre 27/28.03 vs post 03/04.04.2023 around Italy's ChatGPT ban",62 "locator": "arxiv-2304.09339v1, Section 4 Table 1, panel A, row Treat x Post, column (3) Output (Users with Affiliation)",63 "harmonisable": false,64 "harmonised_estimate": null,65 "harmonised_se": null,66 "poolable": false,67 "transformation": "none applied (not harmonisable)",68 "reason_if_not": "Effect on output counts (GitHub events) from a DiD around withdrawal of access, not a comparison of task-completion times with vs without AI on the same tasks, and not randomised."69 }70 },71 "k_extracted": 5,72 "k_harmonised": 4,73 "k_pooled": 2,74 "pooled_studies": [75 "S1",76 "S4"77 ],78 "fe_estimate": -0.18814332871148873,79 "fe_se": 0.10626483095451474,80 "fe_ci_low": -0.39642239738233764,81 "fe_ci_high": 0.02013573995936016,82 "re_estimate": -0.18814332871148873,83 "re_se": 0.10626483095451474,84 "re_ci_low": -0.39642239738233764,85 "re_ci_high": 0.02013573995936016,86 "q_stat": 0.3236860522786328,87 "q_df": 1,88 "q_pvalue": 0.5694008376973132,89 "i2": 0.0,90 "tau2": 0.0,91 "heterogeneity_interpretation": "low",92 "pooled_significant_5pct": false,93 "pooled_pct_change_re": -17.15040485348688,94 "pooled_pct_change_re_ci": [95 -32.72775202989503,96 2.033983151439211597 ],98 "weights_fe": {99 "S1": 0.5761333825403874,100 "S4": 0.4238666174596126101 },102 "weights_re": {103 "S1": 0.5761333825403874,104 "S4": 0.4238666174596126105 },106 "baseline_unweighted_mean_theta_harmonised": -0.13434604282593213,107 "baseline_unweighted_mean_theta_poolable": -0.17882902173411622,108 "sensitivity_s1_se_from_ci": {109 "s1_se": 0.13775510204081634,110 "re_estimate": -0.1891067723193802,111 "re_se": 0.10527305720065441,112 "re_ci_low": -0.39544196443266283,113 "re_ci_high": 0.01722841979390244,114 "q_stat": 0.3281104262022853,115 "i2": 0.0,116 "tau2": 0.0117 },118 "feasibility": "complete",119 "feasibility_reason": null,120 "data_issues": [121 {122 "issue": "S1: Section VI-B reports N=47 (AI) + N=46 (no-AI) = 93 developers, while Table III states n = 96.",123 "handling": "Not resolvable from the extract; reported. sample_as_printed lists both; does not affect theta or SE, which are taken from Table III/Section VI-C."124 },125 {126 "issue": "S1: Section VI-B gives Mean LogToT=4.6 for the AI group and 4.3 for no-AI while stating AI was faster (96 vs 114 min); the log means appear swapped/inconsistent.",127 "handling": "These log means are not used."128 },129 {130 "issue": "S1: Table III is rendered with the four model columns collapsed into one line; SE (0.14) is rounded, and the CI [-0.51, 0.03] implies SE 0.1378.",131 "handling": "Model 2 taken as the second value per the editorial note; printed SE 0.14 used; CI-based SE reported as sensitivity (sensitivity_s1_se_from_ci)."132 },133 {134 "issue": "S2: The uncertainty of the headline +19% estimate is shown only in Figure 1/Figure 15, which are not reproduced; no SE or p-value printed.",135 "handling": "Harmonised (theta = ln 1.19) but not poolable; harmonised_se = null."136 },137 {138 "issue": "S3: Treatment/Control means 44.29/63.10 imply -29.81%, printed improvement is -29.79% (slightly beyond rounding of the printed means); significance shown only by stars with an unusual convention (* = 0.05<p<=0.1).",139 "handling": "Printed -29.79% used as instructed (difference in theta ~0.0003); not poolable because stars do not identify an SE."140 },141 {142 "issue": "S4: Table 3 units ambiguous: Malicious/Non-Malicious times per email (1.8-2.8 's') cannot average to Full Corpus times (50.5/56.9 s) under any common unit; Full Corpus ratio 50.5/56.9 implies 11.25% saving vs printed 11.1% (within rounding).",143 "handling": "Only the Full Corpus column is used; theta depends only on the printed % saving (unit-free). Flagged as a source inconsistency."144 },145 {146 "issue": "S4: SE is approximate, back-calculated from a two-sided p-value (0.471) under a normal approximation on the log scale, although the paper's test is presumably on the time difference.",147 "handling": "Marked as approximate SE in transformation; pooled results carry this caveat."148 },149 {150 "issue": "S5: Table 1 note prints '** p<0.5' (presumably p<0.05); row label rendered as '\\times TreatPost'; panel B header lists 11.04 although the text says it was excluded; rendered date line 'August 24, 2026' is an artefact.",151 "handling": "No effect on the extracted panel A column 3 value; S5 is not harmonisable anyway."152 }153 ],154 "n_input_rows": {155 "studies.json": 5,156 "README.md": 17,157 "sources/arxiv-2410.12944v3.txt": 112,158 "sources/arxiv-2507.09089v2.txt": 132,159 "sources/arxiv-2411.01067v2.txt": 109,160 "sources/arxiv-2511.13860v1.txt": 162,161 "sources/arxiv-2304.09339v1.txt": 108162 },163 "n_analysed": 5,164 "random_seed": null165}