"""T3-synthesis: extraction, harmonisation onto theta = ln(T_AI/T_noAI), pooling, forest plot. Run from the workspace root: ~/wp29/venv/bin/python out/analysis.py Deterministic; uses only files under data/.""" import json, math, os, subprocess import numpy as np, pandas as pd from scipy.stats import norm import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt D = "data"; SRC = os.path.join(D, "sources"); OUT = "out" os.makedirs(OUT, exist_ok=True) def lines(fn): with open(os.path.join(SRC, fn + ".txt"), encoding="utf-8") as f: return f.read().split("\n") def show(fn, ln, must_contain): """Print source line ln (1-based) and verify it contains the expected printed strings.""" L = lines(fn)[ln - 1] for s in must_contain: assert s in L, f"{fn}:{ln} does not contain {s!r}" print(f"[{fn} line {ln}] {L.strip()[:300]}") return L studies = json.load(open(os.path.join(D, "studies.json"))) print("=== Source lines relied upon ===") # S1 Paradis et al. show("arxiv-2410.12944v3", 66, ["TABLE III", "n = 96"]) show("arxiv-2410.12944v3", 76, ["-0.24.", "ExpCon"]) show("arxiv-2410.12944v3", 78, ["(0.14) (0.14) (0.14) (0.17)"]) show("arxiv-2410.12944v3", 58, ["\\beta_{1}=-0.24", "[-0.51,0.03]", "p=0.086"]) show("arxiv-2410.12944v3", 48, ["N=47", "N=46", "Mean LogToT=4.6", "Mean LogToT=4.3"]) # S2 Becker et al. show("arxiv-2507.09089v2", 45, ["completion time with AI allowed"]) show("arxiv-2507.09089v2", 62, ["136 issues", "110 issues", "19% more time", "(Figure 1)"]) show("arxiv-2507.09089v2", 8, ["Figure 1, which displays the confidence interval"]) show("arxiv-2507.09089v2", 72, ["34%", "19% slowdown"]) # S3 Bono & Xu show("arxiv-2411.01067v2", 55, ["All Content | 53.84 | 44.29 | 63.10 | -29.79%***"]) show("arxiv-2411.01067v2", 10, ["asterisks only"]) show("arxiv-2411.01067v2", 38, ["181 subjects"]) # S4 Bono show("arxiv-2511.13860v1", 90, ["Aware Time per Email (s) | 2.8 | 1.8 | 50.5"]) show("arxiv-2511.13860v1", 92, ["Control Time per Email (s) | 1.9 | 2.4 | 56.9"]) show("arxiv-2511.13860v1", 94, ["% Time Savings | -52.9 | 23.6 | 11.1"]) show("arxiv-2511.13860v1", 96, ["p-value | 0.006 | 0.126 | 0.471"]) show("arxiv-2511.13860v1", 31, ["N{=}52"]); show("arxiv-2511.13860v1", 35, ["N{=}56"]) # S5 Kreitmeir & Raschky show("arxiv-2304.09339v1", 80, ["TreatPost | 0.115 | -0.012** | -0.278"]) show("arxiv-2304.09339v1", 82, ["(0.109) | (0.005) | (0.295)"]) show("arxiv-2304.09339v1", 88, ["N | 18332"]) show("arxiv-2304.09339v1", 108, ["** p<0.5"]) print() # ---------------- harmonisation ---------------- items = {} # S1: coefficient on ExpCon in log(time) regression -> theta=b, SE=SE(b) b1, se1 = -0.24, 0.14 se1_ci = (0.03 - (-0.51)) / (2 * 1.96) items["S1"] = dict( estimate_as_printed="ExpCon coefficient (Model 2, LogToT) = -0.24 (marked '.', p<.1); text: effect of AI = 21% faster", uncertainty_as_printed="SE (0.14); 95% CI = [-0.51, 0.03]; p=0.086", sample_as_printed="n = 96 (Table III caption); Section VI-B: N=47 AI, N=46 no-AI", definition="OLS coefficient on experimental condition (AI vs no-AI exposure) with natural-log time on task as dependent variable, controlling for developer-level factors (AvgProgHrsDay, Level); one complex enterprise coding task, randomised", locator="arxiv-2410.12944v3, Table III, row 'ExpCon', column (2) 'Dev' (value -0.24., SE line (0.14)); CI and p-value in Section VI-C text", harmonisable=True, harmonised_estimate=b1, harmonised_se=se1, poolable=True, transformation=f"theta = b = -0.24; SE(theta) = SE(b) = 0.14 (printed). Cross-check from printed CI: (0.03-(-0.51))/(2*1.96) = {se1_ci:.4f}", reason_if_not=None) # S2: percentage change +19% -> theta = ln(1.19); CI only in Figure 1 (not reproduced) c2 = 0.19 items["S2"] = dict( estimate_as_printed="+19% (\"allowing AI actually increases completion time by 19%\"; \"implement issues in 19% more time on average\")", uncertainty_as_printed="none printed in extract (95% CI shown only in Figure 1, not reproduced)", sample_as_printed="16 developers, 246 tasks (136 issues AI-allowed, 110 AI-disallowed)", definition="S = E[completion time with AI allowed]/E[completion time with AI disallowed] - 1, estimated as exp(beta)-1 from log-linear OLS of total implementation time on AI indicator and log forecast time; issue-level randomisation", locator="arxiv-2507.09089v2, Abstract and Section 3 'Results' opening paragraph (19%); estimator in Section 2.3 and Appendix D.1-D.3", harmonisable=True, harmonised_estimate=math.log(1 + c2), harmonised_se=None, poolable=False, transformation=f"theta = ln(1 + c) = ln(1 + 0.19) = {math.log(1+c2):.4f} (equivalently theta = beta-hat, since S = exp(beta)-1)", reason_if_not="No SE, CI or p-value for the 19% estimate is printed in the extract; its confidence interval appears only in Figure 1, which is not reproduced. SE cannot be obtained, so not poolable.") # S3: -29.79% -> theta = ln(1-0.2979); stars only c3 = -0.2979 items["S3"] = dict( estimate_as_printed="-29.79%*** (Treatment 44.29 min, Control 63.10 min; Overall 53.84 min)", uncertainty_as_printed="significance stars only: *** (0 c = -0.111, theta = ln(0.889), SE ~ |theta|/z c4 = -0.111; p4 = 0.471 th4 = math.log(1 + c4); z4 = norm.ppf(1 - p4 / 2); se4 = abs(th4) / z4 items["S4"] = dict( estimate_as_printed="% Time Savings 11.1 (Aware 50.5 s per email, Control 56.9 s per email), Full Corpus", uncertainty_as_printed="p-value 0.471", sample_as_printed="Aware N=56, Control N=52 (167 analysts across three arms; 25-email queues)", definition="Percentage time saving in mean time per email (Aware arm: agent verdicts/rationale and agent-ordered queue vs Control: random order, no AI), across the full queue; randomised", locator="arxiv-2511.13860v1, Section 3.3, Table 3, column 'Full Corpus', rows 'Aware Time per Email (s)', 'Control Time per Email (s)', '% Time Savings', 'p-value'", harmonisable=True, harmonised_estimate=th4, harmonised_se=se4, poolable=True, transformation=f"c = -0.111 (11.1% saving); theta = ln(1 - 0.111) = {th4:.4f}; approximate SE = |theta|/z_(1-0.471/2) = {abs(th4):.4f}/{z4:.4f} = {se4:.4f} (APPROXIMATE SE recovered from p-value). Ratio of printed means: ln(50.5/56.9) = {math.log(50.5/56.9):.4f}", reason_if_not=None) # S5: DiD on output counts around a ban -> not harmonisable items["S5"] = dict( estimate_as_printed="Treat x Post = -0.278 (Output, column 3, panel A, users with affiliation)", uncertainty_as_printed="SE (0.295); no stars", sample_as_printed="N = 18332", definition="Difference-in-differences coefficient on the aggregate GitHub Output measure (sum of Push, Pull Requests, PR comments, commit comments, Create, Issues events; the text describes log counts for these measures) for Italian vs Austrian/French users, first two business days after vs before the ChatGPT ban", locator="arxiv-2304.09339v1, Table 1, panel A (Pre 27/28.03 - Post 03/04.04), row 'Treat x Post', column (3) 'Output'; N from row 'N'", harmonisable=False, harmonised_estimate=None, harmonised_se=None, poolable=False, transformation="none applied (not harmonisable)", reason_if_not="The printed effect is on output counts (GitHub events), not task-completion time on the same tasks, and it comes from a natural experiment (withdrawal of access via a national ban, DiD) rather than a randomised AI-vs-no-AI comparison of completion times. It cannot be mapped onto theta = ln(T_AI/T_noAI).") # ---------------- pooling ---------------- pool = [k for k in items if items[k]["poolable"]] y = np.array([items[k]["harmonised_estimate"] for k in pool]) s = np.array([items[k]["harmonised_se"] for k in pool]) w = 1 / s**2 fe = float(np.sum(w * y) / np.sum(w)); fe_se = float(math.sqrt(1 / np.sum(w))) Q = float(np.sum(w * (y - fe) ** 2)); df = len(pool) - 1 C = float(np.sum(w) - np.sum(w**2) / np.sum(w)) tau2 = max(0.0, (Q - df) / C) i2 = max(0.0, (Q - df) / Q) * 100 if Q > 0 else 0.0 wr = 1 / (s**2 + tau2) re = float(np.sum(wr * y) / np.sum(wr)); re_se = float(math.sqrt(1 / np.sum(wr))) re_lo, re_hi = re - 1.96 * re_se, re + 1.96 * re_se het = "low" if i2 < 25 else ("moderate" if i2 < 50 else "substantial") # sensitivity: S1 SE from CI instead of printed SE s_alt = s.copy(); s_alt[pool.index("S1")] = se1_ci w_alt = 1 / s_alt**2; fe_alt = float(np.sum(w_alt * y) / np.sum(w_alt)) print(f"Sensitivity (S1 SE from CI = {se1_ci:.4f}): FE = {fe_alt:.4f}") def nlines(path): return int(subprocess.run(["wc", "-l", path], capture_output=True, text=True).stdout.split()[0]) n_input = {"studies.json": len(studies), "README.md": nlines(os.path.join(D, "README.md"))} for fn in sorted(os.listdir(SRC)): n_input[fn] = nlines(os.path.join(SRC, fn)) data_issues = [ {"issue": "S1 (arxiv-2410.12944v3, Section VI-B): the AI group is stated to be faster, yet its printed Mean LogToT (4.6) is larger than the no-AI group's (4.3); neither matches the printed raw means (exp(4.6)~99, exp(4.3)~74 vs 96 and 114 min). Internal inconsistency of the source.", "handling": "Not used; the harmonised value comes from the Table III Model 2 coefficient as instructed by studies.json."}, {"issue": "S1: Table III caption says n = 96 while Section VI-B reports N=47 + N=46 = 93 for the raw means.", "handling": "Both sample sizes reported as printed; no effect on theta/SE, which are taken directly from Table III."}, {"issue": "S1: Table III rendering collapses the four model columns into one cell per row, and the R^2 / p-value rows have garbled labels ('R^{2} ... Adj.', '... p value').", "handling": "Column order Models 1-4 left to right per the editorial note; Model 2 ExpCon = -0.24 (0.14) cross-checked against the text (beta_1 = -0.24, CI [-0.51, 0.03], p = 0.086)."}, {"issue": f"S1: two possible SEs: printed SE 0.14 (rounded to 2 dp) and the SE implied by the printed 95% CI, (0.03+0.51)/3.92 = {se1_ci:.4f}.", "handling": f"Primary analysis uses the printed SE 0.14; sensitivity with the CI-based SE reported (FE = {fe_alt:.4f})."}, {"issue": "S2: the 95% CI of the headline 19% estimate is displayed only in Figure 1 (not reproduced); the raw ratio of means (34%) is also printed but is not the regression estimate.", "handling": "Harmonised the regression-based 19% (theta = ln 1.19); SE set to null and study not pooled; 34% not used."}, {"issue": f"S3: the printed improvement -29.79% differs slightly from the ratio of the printed means (44.29/63.10 - 1 = {100*(44.29/63.10-1):.2f}%), presumably due to rounding of the means; sign conventions differ between text ('29.79% reduction') and table ('-29.79%').", "handling": "Used the printed improvement c = -0.2979 per the harmonisation convention; difference in theta < 0.001."}, {"issue": f"S4: Table 3 header says 'Time per Email (s)', but the Malicious/Non-Malicious columns (2.8, 1.8, 1.9, 2.4) are implausible as seconds next to Full Corpus 50.5/56.9 s and do not reproduce their printed % savings (e.g. 1-2.8/1.9 = -47.4% vs printed -52.9); likely minutes and/or rounding. Also 1-50.5/56.9 = {100*(1-50.5/56.9):.2f}% vs printed 11.1.", "handling": "Only the Full Corpus column is used, with the printed % Time Savings (11.1) as the harmonisation input; sign convention: positive savings means less time with AI, so c = -0.111."}, {"issue": "S4: only a two-sided p-value (0.471) is printed for the full-corpus time saving; the test underlying it (likely on raw seconds, not logs) is not described in the extract.", "handling": "Approximate SE = |theta|/z_(1-p/2) as permitted; study pooled with an approximate SE and flagged."}, {"issue": "S5: Table 1 notes print the significance key as '** p<0.5' (presumably p<0.05); the row label renders as '\\times TreatPost'.", "handling": "Recorded as printed; S5 is not harmonisable regardless."}, {"issue": "Stray 'p<' / 'p=' lines and footnote numbers in the extracts (rendering artefacts per data/README.md).", "handling": "Ignored."}, ] res = dict( task_id="T3-synthesis", target_definition="theta = ln(T_AI / T_noAI): natural log of the ratio of mean task-completion time with generative-AI assistance to mean completion time without it, on the same tasks, from a randomised comparison (negative = faster with AI).", items=items, k_extracted=sum(1 for k in items if items[k]["estimate_as_printed"]), k_harmonised=sum(1 for k in items if items[k]["harmonised_estimate"] is not None), k_pooled=len(pool), fe_estimate=fe, fe_se=fe_se, fe_ci_low=fe - 1.96 * fe_se, fe_ci_high=fe + 1.96 * fe_se, re_estimate=re, re_se=re_se, re_ci_low=re_lo, re_ci_high=re_hi, q_stat=Q, i2=i2, tau2=tau2, heterogeneity_interpretation=het, pooled_significant_5pct=bool(re_lo > 0 or re_hi < 0), pooled_pct_change_re=100 * (math.exp(re) - 1), feasibility="complete", feasibility_reason=None, data_issues=data_issues, n_input_rows=n_input, n_analysed=len(items), random_seed=None, ) with open(os.path.join(OUT, "results.json"), "w") as f: json.dump(res, f, indent=2) cols = ["estimate_as_printed", "uncertainty_as_printed", "sample_as_printed", "definition", "locator", "harmonised_estimate", "harmonised_se", "harmonisable", "poolable", "transformation", "reason_if_not"] pd.DataFrame([{"study": k, **{c: items[k][c] for c in cols}} for k in items]).to_csv(os.path.join(OUT, "table1.csv"), index=False) # ---------------- forest plot ---------------- short = {s_["id"]: s_["short"].split(",")[0] for s_ in studies} harm = [k for k in items if items[k]["harmonisable"]] fig, ax = plt.subplots(figsize=(8.5, 4.2)) ypos = list(range(len(harm) + 1, 1, -1)) for yp, k in zip(ypos, harm): t = items[k]["harmonised_estimate"]; se = items[k]["harmonised_se"] if se is not None: ax.errorbar(t, yp, xerr=1.96 * se, fmt="s", color="k", capsize=3) lab = f"{t:+.3f} [{t-1.96*se:+.3f}, {t+1.96*se:+.3f}]" + (" (approx. SE)" if k == "S4" else "") else: ax.plot(t, yp, "o", mfc="white", mec="gray") lab = f"{t:+.3f} (no SE; not pooled)" ax.text(1.02, yp, lab, transform=ax.get_yaxis_transform(), va="center", fontsize=8) ax.fill([re_lo, re, re_hi, re], [1, 1.25, 1, 0.75], color="tab:blue") ax.text(1.02, 1, f"{re:+.3f} [{re_lo:+.3f}, {re_hi:+.3f}]", transform=ax.get_yaxis_transform(), va="center", fontsize=8) ax.axvline(0, color="gray", ls="--", lw=0.8) ax.set_yticks(ypos + [1]); ax.set_yticklabels([f"{k} {short[k]}" for k in harm] + [f"RE pooled (k={len(pool)}: {', '.join(pool)})"]) ax.set_xlabel("theta = ln(T_AI / T_noAI) (negative = faster with AI)") ax.set_title("Harmonised effects of generative-AI access on task-completion time\n(S5 not harmonisable; open circles: no SE available)", fontsize=10) ax.set_ylim(0.3, len(harm) + 1.7) plt.tight_layout(); plt.savefig(os.path.join(OUT, "figure1.png"), dpi=150, bbox_inches="tight") print(json.dumps({k: v for k, v in res.items() if k not in ("items", "data_issues")}, indent=2)) for k in items: print(k, items[k]["harmonised_estimate"], items[k]["harmonised_se"], items[k]["poolable"])