1{2 "task_id": "T3-synthesis",3 "target_definition": "theta = ln(T_AI / T_noAI): natural log of the ratio of mean task-completion time with generative-AI assistance to mean completion time without it, on the same tasks, from a randomised comparison (negative = faster with AI).",4 "items": {5 "S1": {6 "estimate_as_printed": "ExpCon (Model 2) = -0.24 (marker '.', p<.1) on LogToT",7 "uncertainty_as_printed": "SE (0.14) [Table III, Model 2]; 95%CI = [-0.51,0.03], p=0.086 [Section VI-C]",8 "sample_as_printed": "n = 96 (Table III caption); N=47 AI, N=46 no-AI in Section VI-B",9 "definition": "OLS coefficient on experimental condition (AI vs no-AI) with natural-log time on task as dependent variable, controlling for developer-level factors (Model 2)",10 "locator": "arxiv-2410.12944v3, Table III (line 76: ExpCon row, 2nd value = Model 2; line 78: SE row), Section VI-C (line 58)",11 "harmonisable": true,12 "harmonised_estimate": -0.24,13 "harmonised_se": 0.14,14 "poolable": true,15 "transformation": "theta = b = -0.24; SE(theta) = SE(b) = 0.14 (printed). Cross-check from CI: (0.03 - (-0.51))/(2*1.96) = 0.1378",16 "reason_if_not": null17 },18 "S2": {19 "estimate_as_printed": "19% more time (issue completion time, AI allowed vs disallowed; regression-based)",20 "uncertainty_as_printed": "none printed in extract (95% CI using HC3 SEs is shown only in Figure 1, not reproduced)",21 "sample_as_printed": "16 developers; 246 tasks (136 issues AI-allowed, 110 AI-disallowed)",22 "definition": "S = E[time AI allowed]/E[time AI disallowed] - 1, estimated as exp(beta)-1 from log-linear OLS of completion time on AI indicator and log forecast time",23 "locator": "arxiv-2507.09089v2, Section 3 Results (line 62); Abstract (line 34); estimator Sections 2.3, D.1-D.3; CI method D.2 (line 97)",24 "harmonisable": true,25 "harmonised_estimate": 0.17395330712343798,26 "harmonised_se": null,27 "poolable": false,28 "transformation": "theta = ln(1 + c) = ln(1 + 0.19) = 0.1740",29 "reason_if_not": "Not poolable: no SE, CI or p-value for the 19% estimate is printed in the extract; the confidence interval appears only in Figure 1 (not reproduced), so SE(theta) cannot be obtained."30 },31 "S3": {32 "estimate_as_printed": "All Content (aggregate): Overall 53.84, Treatment 44.29, Control 63.10 minutes; Improvement -29.79%***",33 "uncertainty_as_printed": "significance stars only (*** = 0<p<=0.01, per editorial note); no SE or CI printed",34 "sample_as_printed": "181 subjects completed the task (Section 3); treatment/control split not printed",35 "definition": "Percentage difference in mean task duration (minutes), Copilot vs control, holding accuracy constant (control duration predicted at treatment accuracy; Appendix B); bootstrap significance",36 "locator": "arxiv-2411.01067v2, Table 3 (line 55: 'Aggregate Results / All Content' row; columns Treatment, Control, Improvement)",37 "harmonisable": true,38 "harmonised_estimate": -0.353679434958934,39 "harmonised_se": null,40 "poolable": false,41 "transformation": "theta = ln(1 + c) = ln(1 + (-0.2979)) = -0.3537 (cross-check ln(44.29/63.10) = -0.3540)",42 "reason_if_not": "Not poolable: uncertainty is reported only as significance stars (***, p<=0.01); a p-value threshold does not identify SE(theta)."43 },44 "S4": {45 "estimate_as_printed": "Full Corpus: Aware Time per Email (s) 50.5; Control Time per Email (s) 56.9; % Time Savings 11.1",46 "uncertainty_as_printed": "p-value 0.471",47 "sample_as_printed": "Aware N=56, Control N=52 (Blind N=59 not used); 25-email queue per subject",48 "definition": "Percentage time saving in mean processing time per email (full queue), Aware arm (agent verdicts + rationale + priority ordering) vs Control (no AI)",49 "locator": "arxiv-2511.13860v1, Section 3.3, Table 3 (lines 90-96; column 'Full Corpus')",50 "harmonisable": true,51 "harmonised_estimate": -0.11765804346823246,52 "harmonised_se": 0.1632205919720678,53 "poolable": true,54 "transformation": "c = -11.1/100 = -0.111; theta = ln(1 + c) = ln(0.889) = -0.1177 (cross-check ln(50.5/56.9) = -0.1193); APPROXIMATE SE = |theta|/z_(1-p/2) = 0.1177/0.7209 = 0.1632 from printed p = 0.471",55 "reason_if_not": null56 },57 "S5": {58 "estimate_as_printed": "Treat x Post (Output, column 3, panel A) = -0.278",59 "uncertainty_as_printed": "SE (0.295) (clustered by user); no significance star",60 "sample_as_printed": "N = 18332 (user-days, users with affiliation, panel A)",61 "definition": "Difference-in-differences coefficient on an aggregate GitHub output count (Push, PRs, comments, Create, Issues) for Italian vs Austrian/French users, around the ChatGPT ban (withdrawal of access)",62 "locator": "arxiv-2304.09339v1, Section 4, Table 1, panel A, row 'Treat x Post' (line 80), SE line 82, N line 88, column (3) Output",63 "harmonisable": false,64 "harmonised_estimate": null,65 "harmonised_se": null,66 "poolable": false,67 "transformation": "none (not harmonisable)",68 "reason_if_not": "Not harmonisable: the outcome is a count of GitHub output events per user-day (activity volume), not task-completion time on the same tasks; the contrast is country-level withdrawal of access (DiD), not a randomised AI-vs-no-AI comparison of time on identical tasks."69 }70 },71 "k_extracted": 5,72 "k_harmonised": 4,73 "k_pooled": 2,74 "fe_estimate": -0.18814332871148873,75 "fe_se": 0.10626483095451474,76 "fe_ci_low": -0.39642239738233764,77 "fe_ci_high": 0.02013573995936016,78 "re_estimate": -0.18814332871148873,79 "re_se": 0.10626483095451474,80 "re_ci_low": -0.39642239738233764,81 "re_ci_high": 0.02013573995936016,82 "q_stat": 0.3236860522786328,83 "i2": 0.0,84 "tau2": 0.0,85 "heterogeneity_interpretation": "low",86 "pooled_significant_5pct": false,87 "pooled_pct_change_re": -17.15040485348688,88 "feasibility": "complete",89 "feasibility_reason": null,90 "data_issues": [91 {92 "issue": "S1 (arxiv-2410.12944v3, Section VI-B): the printed mean LogToT is 4.6 for the AI group and 4.3 for the no-AI group, which contradicts the printed mean times (96 min AI vs 114 min no-AI; ln 96 = 4.56, ln 114 = 4.74) and the statement that the AI group was faster.",93 "handling": "Not used. The harmonised value is the Model 2 regression coefficient specified in studies.json, whose sign agrees with the mean times and with the text."94 },95 {96 "issue": "S1: Table III caption says n = 96, whereas Section VI-B reports N=47 (AI) + N=46 (no-AI) = 93 for the time comparison.",97 "handling": "Both reported as printed in sample_as_printed; the SE is taken as printed, so the discrepancy does not affect theta or SE."98 },99 {100 "issue": "S1: Table III renders the four model columns collapsed into one line, with significance markers attached to the numbers (e.g. '-0.24.'). The trailing '.' is the p<.1 marker, not a decimal point. The printed SE (0.14) and the SE implied by the printed 95% CI [-0.51, 0.03] (0.1378) differ slightly because of rounding.",101 "handling": "Model 2 = second value on the ExpCon line; the marker was stripped. The printed SE 0.14 was used, and a sensitivity analysis uses the CI-derived SE."102 },103 {104 "issue": "S1: the extract has stray 'p<' / 'p=' lines and footnote digits ('^2^2', '2').",105 "handling": "Ignored as rendering artefacts (data/README.md)."106 },107 {108 "issue": "S2: the uncertainty of the headline 19% estimate is shown only in Figure 1 (not reproduced). The raw ratio of means (34%) differs from the regression-based 19%.",109 "handling": "Used the regression-based 19%, as studies.json specifies. Harmonised theta = ln(1.19). Not poolable because there is no SE."110 },111 {112 "issue": "S3: the Table 3 header row has four labels ('Overall | Treatment | Control | Improvement') with an empty leading cell. The 'Improvement' column is actually a signed % change in time (negative = faster); for MC rows a positive value is a time increase (per the text). The printed -29.79% matches 44.29/63.10 - 1 = -0.2981 only up to rounding.",113 "handling": "Read the 'Improvement' value as the signed % change c = -0.2979. Computed theta from the printed percentage and reported ln(44.29/63.10) as a cross-check."114 },115 {116 "issue": "S3: the significance-star convention is non-standard (* 0.05<p<=0.1, ** 0.01<p<=0.05, *** p<=0.01).",117 "handling": "Recorded. Stars give only a p-value threshold, so S3 is not poolable."118 },119 {120 "issue": "S4: Table 3 labels all columns 'Time per Email (s)', but the Malicious (2.8 / 1.9) and Non-Malicious (1.8 / 2.4) values cannot be seconds if the Full Corpus value is 50.5 / 56.9 s. No mix of the two categories gives the full-corpus mean in either seconds or minutes, so the units and/or rows are internally inconsistent.",121 "handling": "Only the Full Corpus column is used, and theta depends only on the unit-free ratio and % saving. The printed 11.1% agrees with 1 - 50.5/56.9 = 11.25% within rounding of the printed means. Flagged."122 },123 {124 "issue": "S4: the 'Aware' arm differs from Control in queue ordering as well as AI verdicts/rationales, and its % time saving is printed positive for a decrease in time (sign opposite to theta).",125 "handling": "Aware vs Control is taken as the AI vs no-AI contrast, and the sign is converted (c = -0.111). The SE is approximate, back-calculated from p = 0.471."126 },127 {128 "issue": "S5: the Table 1 footnote prints '** p<0.5' (presumably p<0.05). The extract's date line 'August 24, 2026' is a rendering-time artefact.",129 "handling": "Recorded. This does not affect anything because S5 is not harmonisable."130 }131 ],132 "n_input_rows": {133 "studies.json": 5,134 "README.md": 17,135 "sources/arxiv-2304.09339v1.txt": 108,136 "sources/arxiv-2410.12944v3.txt": 112,137 "sources/arxiv-2411.01067v2.txt": 109,138 "sources/arxiv-2507.09089v2.txt": 132,139 "sources/arxiv-2511.13860v1.txt": 162140 },141 "n_analysed": 5,142 "random_seed": null,143 "sensitivity_s1_se_from_ci": {144 "s1_se": 0.13775510204081634,145 "fe_estimate": -0.1891067723193802,146 "fe_se": 0.10527305720065441,147 "re_estimate": -0.1891067723193802,148 "re_se": 0.10527305720065441,149 "q_stat": 0.3281104262022853,150 "i2": 0.0,151 "tau2": 0.0152 }153}