1{2 "task_id": "T3-synthesis",3 "target_definition": "theta = ln(T_AI / T_noAI): natural log of the ratio of mean task-completion time with access to a generative-AI assistant to mean completion time without it, for the same tasks, from a randomised comparison; negative values mean faster with AI.",4 "items": {5 "S1": {6 "estimate_as_printed": "-0.24 (ExpCon, Model 2, LogToT; marker '.'); text: beta_1=-0.24, effect of AI = 21%",7 "uncertainty_as_printed": "SE (0.14); 95% CI = [-0.51, 0.03]; p=0.086",8 "sample_as_printed": "Table III: n = 96 (Section VI-B: N=47 AI, N=46 no-AI)",9 "definition": "OLS coefficient of the experimental condition (AI vs no-AI, randomised) on the natural log of time on task, controlling for developer-level factors (Model 2)",10 "locator": "arxiv-2410.12944v3; Table III (L76-L78), row ExpCon, column (2) 'Dev'; Section VI-C (L58) for CI and p",11 "harmonisable": true,12 "harmonised_estimate": -0.24,13 "harmonised_se": 0.14,14 "poolable": true,15 "transformation": "log-time coefficient: theta = b = -0.24; SE(theta) = SE(b) = 0.14 (printed SE used; CI-based SE = (0.03 - (-0.51))/3.92 = 0.1378 used only as sensitivity)",16 "reason_if_not": null17 },18 "S2": {19 "estimate_as_printed": "19% more time on average (AI allowed increases completion time by 19%)",20 "uncertainty_as_printed": "none printed (confidence interval shown only in Figure 1 / Figure 15, not reproduced in the extract)",21 "sample_as_printed": "16 developers, 246 tasks; 136 issues AI-allowed, 110 issues AI-disallowed",22 "definition": "S = E[time with AI allowed]/E[time AI disallowed] - 1, estimated as exp(beta)-1 from OLS of log implementation time on AI indicator and log forecast time (issue-level randomisation)",23 "locator": "arxiv-2507.09089v2; Section 3 Results (L62); abstract (L34); estimator in Section 2.3 (L45) and Appendix D.1-D.3",24 "harmonisable": true,25 "harmonised_estimate": 0.17395330712343798,26 "harmonised_se": null,27 "poolable": false,28 "transformation": "percentage change: c = +0.19; theta = ln(1 + 0.19) = 0.173953 (equals the regression beta since S = exp(beta) - 1)",29 "reason_if_not": "Not poolable: no SE, CI or p-value is printed in the extract; the confidence interval appears only in a figure that is not reproduced, so SE(theta) cannot be obtained."30 },31 "S3": {32 "estimate_as_printed": "Treatment 44.29 min, Control 63.10 min (Overall 53.84); Improvement -29.79%***",33 "uncertainty_as_printed": "significance stars only (***; per the paper's scheme 0<p<=0.01); no SE or CI printed",34 "sample_as_printed": "181 subjects completed the task satisfactorily (one subject dropped)",35 "definition": "Percentage difference in mean task duration (minutes), Copilot (treatment) vs control without Copilot, holding accuracy constant (control duration predicted at treatment accuracy; Appendix B), randomised",36 "locator": "arxiv-2411.01067v2; Section 4.2, Table 3 (L51-L55), row 'Aggregate Results / All Content', columns Treatment, Control, Improvement",37 "harmonisable": true,38 "harmonised_estimate": -0.353679434958934,39 "harmonised_se": null,40 "poolable": false,41 "transformation": "percentage change: c = -0.2979; theta = ln(1 + (-0.2979)) = -0.353679 (ratio of displayed means ln(44.29/63.10) = -0.353962, sensitivity only)",42 "reason_if_not": "Not poolable: uncertainty is printed only as significance stars (p<=0.01); SE(theta) cannot be recovered from a threshold."43 },44 "S4": {45 "estimate_as_printed": "Aware 50.5, Control 56.9 (row labels: Time per Email (s)); % Time Savings 11.1",46 "uncertainty_as_printed": "p-value 0.471",47 "sample_as_printed": "Aware N=56, Control N=52 (of 167 analysts in three arms; 13 low-scoring responses filtered)",48 "definition": "Percentage time saving per processed email (Full Corpus, whole 25-email queue without resolve-benign protocol), Aware arm (agent verdicts, rationale, agent-ordered queue) vs Control (random order, no AI), randomised",49 "locator": "arxiv-2511.13860v1; Section 3.3, Table 3 (L86-L96), column 'Full Corpus', rows Aware/Control Time per Email, % Time Savings, p-value",50 "harmonisable": true,51 "harmonised_estimate": -0.11765804346823246,52 "harmonised_se": 0.1632205919720678,53 "poolable": true,54 "transformation": "percentage time saving 11.1% -> c = -0.111; theta = ln(1 + (-0.111)) = -0.117658; APPROXIMATE SE from two-sided p: SE = |theta|/z_(1-0.471/2) = 0.117658/0.720853 = 0.163221 (ratio of displayed means ln(50.5/56.9) = -0.119322, sensitivity only)",55 "reason_if_not": null56 },57 "S5": {58 "estimate_as_printed": "-0.278 (Treat x Post, Output, panel A, column (3))",59 "uncertainty_as_printed": "SE (0.295) (clustered at user level)",60 "sample_as_printed": "N = 18332 (user-day observations, users with affiliation)",61 "definition": "Difference-in-differences coefficient (Italy vs Austria/France, post vs pre ChatGPT ban) on an aggregate count of GitHub Output events (Push, Pull Requests, comments, Create, Issues)",62 "locator": "arxiv-2304.09339v1; Section 4, Table 1 (L80-L88), panel A 'Pre 27/28.03 - Post 03/04.04', row Treat x Post, column (3) Output (users with affiliation)",63 "harmonisable": false,64 "harmonised_estimate": null,65 "harmonised_se": null,66 "poolable": false,67 "transformation": "none: no transformation maps an effect on output counts onto ln(T_AI/T_noAI)",68 "reason_if_not": "Not harmonisable: the effect is on output counts (GitHub events) around the withdrawal of ChatGPT access in a non-randomised difference-in-differences design, not a comparison of task-completion times with and without AI on the same tasks."69 }70 },71 "k_extracted": 5,72 "k_harmonised": 4,73 "k_pooled": 2,74 "pooled_study_ids": [75 "S1",76 "S4"77 ],78 "fe_estimate": -0.18814332871148873,79 "fe_se": 0.10626483095451474,80 "fe_ci_low": -0.39642239738233764,81 "fe_ci_high": 0.02013573995936016,82 "re_estimate": -0.18814332871148873,83 "re_se": 0.10626483095451474,84 "re_ci_low": -0.39642239738233764,85 "re_ci_high": 0.02013573995936016,86 "q_stat": 0.3236860522786328,87 "q_df": 1,88 "i2": 0.0,89 "tau2": 0.0,90 "heterogeneity_interpretation": "low",91 "pooled_significant_5pct": false,92 "pooled_pct_change_re": -17.15040485348688,93 "pooled_pct_change_re_ci_low": -32.72775202989503,94 "pooled_pct_change_re_ci_high": 2.0339831514392115,95 "baseline_unweighted_mean_theta_harmonised": -0.13434604282593213,96 "baseline_unweighted_mean_theta_pooled": -0.17882902173411622,97 "sensitivity_ci_se_s1_re_estimate": -0.1891067723193802,98 "sensitivity_ci_se_s1_re_se": 0.10527305720065441,99 "sensitivity_ratio_s4_theta": -0.11932200485097105,100 "sensitivity_ratio_s4_re_estimate": -0.18967451746361488,101 "sensitivity_ratio_s4_re_se": 0.10689411567852646,102 "feasibility": "complete",103 "feasibility_reason": null,104 "data_issues": [105 {106 "issue": "S1 (Paradis): Table III states n = 96, while Section VI-B reports N=47 (AI) + N=46 (no-AI) = 93 completed tasks.",107 "handling": "Both sample statements recorded in sample_as_printed; coefficient and SE taken from Table III as designated; no effect on theta."108 },109 {110 "issue": "S1 (Paradis): Section VI-B says the AI group (Mean LogToT=4.6) was faster than the no-AI group (Mean LogToT=4.3), which contradicts the raw means (96 vs 114 min); the log means appear swapped.",111 "handling": "Not used; harmonisation relies on the Model 2 regression coefficient designated in studies.json."112 },113 {114 "issue": "S1 (Paradis): printed SE 0.14 and printed 95% CI [-0.51, 0.03] imply slightly different SEs (0.14 vs 0.1378); '21% faster' denotes a 21% reduction in time (1-exp(beta)).",115 "handling": "Printed SE used (convention for a coefficient with SE); CI-based SE reported as sensitivity_ci_se_s1_* keys."116 },117 {118 "issue": "S2 (Becker): the confidence interval for the 19% estimate is shown only in Figure 1/Figure 15, which are not reproduced in the extract.",119 "handling": "No SE imputed; S2 harmonised (theta = ln 1.19) but not poolable."120 },121 {122 "issue": "S3 (Bono & Xu): uncertainty only as significance stars with a non-standard scheme (* 0.05<p<=0.1, ** 0.01<p<=0.05, *** p<=0.01); displayed minutes give 44.29/63.10-1 = -29.81% vs printed -29.79%; control subjects could use other AI tools (25% did).",123 "handling": "Printed -29.79% used for theta; S3 not poolable; contamination noted as a limitation."124 },125 {126 "issue": "S4 (Bono): printed % Time Savings 11.1 does not equal 1-50.5/56.9 = 11.25% (Malicious and Non-Malicious columns also do not reproduce from displayed times); row labels say seconds per email, but Full Corpus values (50.5, 56.9) are inconsistent with the per-email Malicious/Non-Malicious values (2.8/1.9, 1.8/2.4), so the unit of the Full Corpus column is ambiguous.",127 "handling": "Printed % saving (11.1) used for theta (unit-free); ratio of displayed means used as sensitivity (sensitivity_ratio_s4_* keys); SE recovered approximately from p = 0.471."128 },129 {130 "issue": "S4 (Bono): the Aware arm differs from Control by both agent verdicts/rationales and agent-prioritised queue ordering; only a p-value is printed.",131 "handling": "Treated as AI vs no-AI; SE flagged as approximate."132 },133 {134 "issue": "S5 (Kreitmeir & Raschky): table notes print '** p<0.5' (presumably 0.05); row label rendered as '\\times TreatPost'; date line under the author block is a rendering artefact; whether Output is logged is ambiguous in Section 3.1.",135 "handling": "Values copied verbatim; S5 not harmonisable regardless of these issues."136 },137 {138 "issue": "Stray 'p<', 'p=' and footnote-number lines in the extracts.",139 "handling": "Ignored as rendering artefacts per data/README.md."140 }141 ],142 "n_input_rows": {143 "studies.json": 5,144 "README.md": 17,145 "sources/arxiv-2304.09339v1.txt": 108,146 "sources/arxiv-2410.12944v3.txt": 112,147 "sources/arxiv-2411.01067v2.txt": 109,148 "sources/arxiv-2507.09089v2.txt": 132,149 "sources/arxiv-2511.13860v1.txt": 162150 },151 "n_analysed": 5,152 "random_seed": null153}