1"""T3-synthesis: extraction, harmonisation onto theta = ln(T_AI/T_noAI), pooling, forest plot.2Run from the workspace root: ~/wp29/venv/bin/python out/analysis.py3Deterministic; uses only files under data/."""4import json, math, os, subprocess5import numpy as np, pandas as pd6from scipy.stats import norm7import matplotlib8matplotlib.use("Agg")9import matplotlib.pyplot as plt1011D = "data"; SRC = os.path.join(D, "sources"); OUT = "out"12os.makedirs(OUT, exist_ok=True)1314def lines(fn):15 with open(os.path.join(SRC, fn + ".txt"), encoding="utf-8") as f:16 return f.read().split("\n")1718def show(fn, ln, must_contain):19 """Print source line ln (1-based) and verify it contains the expected printed strings."""20 L = lines(fn)[ln - 1]21 for s in must_contain:22 assert s in L, f"{fn}:{ln} does not contain {s!r}"23 print(f"[{fn} line {ln}] {L.strip()[:300]}")24 return L2526studies = json.load(open(os.path.join(D, "studies.json")))27print("=== Source lines relied upon ===")28# S1 Paradis et al.29show("arxiv-2410.12944v3", 66, ["TABLE III", "n = 96"])30show("arxiv-2410.12944v3", 76, ["-0.24.", "ExpCon"])31show("arxiv-2410.12944v3", 78, ["(0.14) (0.14) (0.14) (0.17)"])32show("arxiv-2410.12944v3", 58, ["\\beta_{1}=-0.24", "[-0.51,0.03]", "p=0.086"])33show("arxiv-2410.12944v3", 48, ["N=47", "N=46", "Mean LogToT=4.6", "Mean LogToT=4.3"])34# S2 Becker et al.35show("arxiv-2507.09089v2", 45, ["completion time with AI allowed"])36show("arxiv-2507.09089v2", 62, ["136 issues", "110 issues", "19% more time", "(Figure 1)"])37show("arxiv-2507.09089v2", 8, ["Figure 1, which displays the confidence interval"])38show("arxiv-2507.09089v2", 72, ["34%", "19% slowdown"])39# S3 Bono & Xu40show("arxiv-2411.01067v2", 55, ["All Content | 53.84 | 44.29 | 63.10 | -29.79%***"])41show("arxiv-2411.01067v2", 10, ["asterisks only"])42show("arxiv-2411.01067v2", 38, ["181 subjects"])43# S4 Bono44show("arxiv-2511.13860v1", 90, ["Aware Time per Email (s) | 2.8 | 1.8 | 50.5"])45show("arxiv-2511.13860v1", 92, ["Control Time per Email (s) | 1.9 | 2.4 | 56.9"])46show("arxiv-2511.13860v1", 94, ["% Time Savings | -52.9 | 23.6 | 11.1"])47show("arxiv-2511.13860v1", 96, ["p-value | 0.006 | 0.126 | 0.471"])48show("arxiv-2511.13860v1", 31, ["N{=}52"]); show("arxiv-2511.13860v1", 35, ["N{=}56"])49# S5 Kreitmeir & Raschky50show("arxiv-2304.09339v1", 80, ["TreatPost | 0.115 | -0.012** | -0.278"])51show("arxiv-2304.09339v1", 82, ["(0.109) | (0.005) | (0.295)"])52show("arxiv-2304.09339v1", 88, ["N | 18332"])53show("arxiv-2304.09339v1", 108, ["** p<0.5"])54print()5556# ---------------- harmonisation ----------------57items = {}58# S1: coefficient on ExpCon in log(time) regression -> theta=b, SE=SE(b)59b1, se1 = -0.24, 0.1460se1_ci = (0.03 - (-0.51)) / (2 * 1.96)61items["S1"] = dict(62 estimate_as_printed="ExpCon coefficient (Model 2, LogToT) = -0.24 (marked '.', p<.1); text: effect of AI = 21% faster",63 uncertainty_as_printed="SE (0.14); 95% CI = [-0.51, 0.03]; p=0.086",64 sample_as_printed="n = 96 (Table III caption); Section VI-B: N=47 AI, N=46 no-AI",65 definition="OLS coefficient on experimental condition (AI vs no-AI exposure) with natural-log time on task as dependent variable, controlling for developer-level factors (AvgProgHrsDay, Level); one complex enterprise coding task, randomised",66 locator="arxiv-2410.12944v3, Table III, row 'ExpCon', column (2) 'Dev' (value -0.24., SE line (0.14)); CI and p-value in Section VI-C text",67 harmonisable=True, harmonised_estimate=b1, harmonised_se=se1, poolable=True,68 transformation=f"theta = b = -0.24; SE(theta) = SE(b) = 0.14 (printed). Cross-check from printed CI: (0.03-(-0.51))/(2*1.96) = {se1_ci:.4f}",69 reason_if_not=None)70# S2: percentage change +19% -> theta = ln(1.19); CI only in Figure 1 (not reproduced)71c2 = 0.1972items["S2"] = dict(73 estimate_as_printed="+19% (\"allowing AI actually increases completion time by 19%\"; \"implement issues in 19% more time on average\")",74 uncertainty_as_printed="none printed in extract (95% CI shown only in Figure 1, not reproduced)",75 sample_as_printed="16 developers, 246 tasks (136 issues AI-allowed, 110 AI-disallowed)",76 definition="S = E[completion time with AI allowed]/E[completion time with AI disallowed] - 1, estimated as exp(beta)-1 from log-linear OLS of total implementation time on AI indicator and log forecast time; issue-level randomisation",77 locator="arxiv-2507.09089v2, Abstract and Section 3 'Results' opening paragraph (19%); estimator in Section 2.3 and Appendix D.1-D.3",78 harmonisable=True, harmonised_estimate=math.log(1 + c2), harmonised_se=None, poolable=False,79 transformation=f"theta = ln(1 + c) = ln(1 + 0.19) = {math.log(1+c2):.4f} (equivalently theta = beta-hat, since S = exp(beta)-1)",80 reason_if_not="No SE, CI or p-value for the 19% estimate is printed in the extract; its confidence interval appears only in Figure 1, which is not reproduced. SE cannot be obtained, so not poolable.")81# S3: -29.79% -> theta = ln(1-0.2979); stars only82c3 = -0.297983items["S3"] = dict(84 estimate_as_printed="-29.79%*** (Treatment 44.29 min, Control 63.10 min; Overall 53.84 min)",85 uncertainty_as_printed="significance stars only: *** (0<p<=0.01)",86 sample_as_printed="181 subjects completed the task satisfactorily (treatment/control split not printed in extract)",87 definition="Percentage difference in mean task duration (minutes), Copilot vs control, holding accuracy constant (control duration predicted at treatment accuracy via a control-group regression of duration on accuracy); all content, randomised",88 locator="arxiv-2411.01067v2, Table 3, block 'Aggregate Results', row 'All Content', columns Treatment / Control / Improvement",89 harmonisable=True, harmonised_estimate=math.log(1 + c3), harmonised_se=None, poolable=False,90 transformation=f"theta = ln(1 + c) = ln(1 - 0.2979) = {math.log(1+c3):.4f} (ratio of printed means: ln(44.29/63.10) = {math.log(44.29/63.10):.4f})",91 reason_if_not="Only significance stars (***, p<=0.01) are printed; no SE, CI or exact p-value, so SE cannot be obtained and the study is not poolable.")92# S4: % time savings 11.1, p=0.471 -> c = -0.111, theta = ln(0.889), SE ~ |theta|/z93c4 = -0.111; p4 = 0.47194th4 = math.log(1 + c4); z4 = norm.ppf(1 - p4 / 2); se4 = abs(th4) / z495items["S4"] = dict(96 estimate_as_printed="% Time Savings 11.1 (Aware 50.5 s per email, Control 56.9 s per email), Full Corpus",97 uncertainty_as_printed="p-value 0.471",98 sample_as_printed="Aware N=56, Control N=52 (167 analysts across three arms; 25-email queues)",99 definition="Percentage time saving in mean time per email (Aware arm: agent verdicts/rationale and agent-ordered queue vs Control: random order, no AI), across the full queue; randomised",100 locator="arxiv-2511.13860v1, Section 3.3, Table 3, column 'Full Corpus', rows 'Aware Time per Email (s)', 'Control Time per Email (s)', '% Time Savings', 'p-value'",101 harmonisable=True, harmonised_estimate=th4, harmonised_se=se4, poolable=True,102 transformation=f"c = -0.111 (11.1% saving); theta = ln(1 - 0.111) = {th4:.4f}; approximate SE = |theta|/z_(1-0.471/2) = {abs(th4):.4f}/{z4:.4f} = {se4:.4f} (APPROXIMATE SE recovered from p-value). Ratio of printed means: ln(50.5/56.9) = {math.log(50.5/56.9):.4f}",103 reason_if_not=None)104# S5: DiD on output counts around a ban -> not harmonisable105items["S5"] = dict(106 estimate_as_printed="Treat x Post = -0.278 (Output, column 3, panel A, users with affiliation)",107 uncertainty_as_printed="SE (0.295); no stars",108 sample_as_printed="N = 18332",109 definition="Difference-in-differences coefficient on the aggregate GitHub Output measure (sum of Push, Pull Requests, PR comments, commit comments, Create, Issues events; the text describes log counts for these measures) for Italian vs Austrian/French users, first two business days after vs before the ChatGPT ban",110 locator="arxiv-2304.09339v1, Table 1, panel A (Pre 27/28.03 - Post 03/04.04), row 'Treat x Post', column (3) 'Output'; N from row 'N'",111 harmonisable=False, harmonised_estimate=None, harmonised_se=None, poolable=False,112 transformation="none applied (not harmonisable)",113 reason_if_not="The printed effect is on output counts (GitHub events), not task-completion time on the same tasks, and it comes from a natural experiment (withdrawal of access via a national ban, DiD) rather than a randomised AI-vs-no-AI comparison of completion times. It cannot be mapped onto theta = ln(T_AI/T_noAI).")114115# ---------------- pooling ----------------116pool = [k for k in items if items[k]["poolable"]]117y = np.array([items[k]["harmonised_estimate"] for k in pool])118s = np.array([items[k]["harmonised_se"] for k in pool])119w = 1 / s**2120fe = float(np.sum(w * y) / np.sum(w)); fe_se = float(math.sqrt(1 / np.sum(w)))121Q = float(np.sum(w * (y - fe) ** 2)); df = len(pool) - 1122C = float(np.sum(w) - np.sum(w**2) / np.sum(w))123tau2 = max(0.0, (Q - df) / C)124i2 = max(0.0, (Q - df) / Q) * 100 if Q > 0 else 0.0125wr = 1 / (s**2 + tau2)126re = float(np.sum(wr * y) / np.sum(wr)); re_se = float(math.sqrt(1 / np.sum(wr)))127re_lo, re_hi = re - 1.96 * re_se, re + 1.96 * re_se128het = "low" if i2 < 25 else ("moderate" if i2 < 50 else "substantial")129130# sensitivity: S1 SE from CI instead of printed SE131s_alt = s.copy(); s_alt[pool.index("S1")] = se1_ci132w_alt = 1 / s_alt**2; fe_alt = float(np.sum(w_alt * y) / np.sum(w_alt))133print(f"Sensitivity (S1 SE from CI = {se1_ci:.4f}): FE = {fe_alt:.4f}")134135def nlines(path):136 return int(subprocess.run(["wc", "-l", path], capture_output=True, text=True).stdout.split()[0])137138n_input = {"studies.json": len(studies), "README.md": nlines(os.path.join(D, "README.md"))}139for fn in sorted(os.listdir(SRC)):140 n_input[fn] = nlines(os.path.join(SRC, fn))141142data_issues = [143 {"issue": "S1 (arxiv-2410.12944v3, Section VI-B): the AI group is stated to be faster, yet its printed Mean LogToT (4.6) is larger than the no-AI group's (4.3); neither matches the printed raw means (exp(4.6)~99, exp(4.3)~74 vs 96 and 114 min). Internal inconsistency of the source.",144 "handling": "Not used; the harmonised value comes from the Table III Model 2 coefficient as instructed by studies.json."},145 {"issue": "S1: Table III caption says n = 96 while Section VI-B reports N=47 + N=46 = 93 for the raw means.",146 "handling": "Both sample sizes reported as printed; no effect on theta/SE, which are taken directly from Table III."},147 {"issue": "S1: Table III rendering collapses the four model columns into one cell per row, and the R^2 / p-value rows have garbled labels ('R^{2} ... Adj.', '... p value').",148 "handling": "Column order Models 1-4 left to right per the editorial note; Model 2 ExpCon = -0.24 (0.14) cross-checked against the text (beta_1 = -0.24, CI [-0.51, 0.03], p = 0.086)."},149 {"issue": f"S1: two possible SEs: printed SE 0.14 (rounded to 2 dp) and the SE implied by the printed 95% CI, (0.03+0.51)/3.92 = {se1_ci:.4f}.",150 "handling": f"Primary analysis uses the printed SE 0.14; sensitivity with the CI-based SE reported (FE = {fe_alt:.4f})."},151 {"issue": "S2: the 95% CI of the headline 19% estimate is displayed only in Figure 1 (not reproduced); the raw ratio of means (34%) is also printed but is not the regression estimate.",152 "handling": "Harmonised the regression-based 19% (theta = ln 1.19); SE set to null and study not pooled; 34% not used."},153 {"issue": f"S3: the printed improvement -29.79% differs slightly from the ratio of the printed means (44.29/63.10 - 1 = {100*(44.29/63.10-1):.2f}%), presumably due to rounding of the means; sign conventions differ between text ('29.79% reduction') and table ('-29.79%').",154 "handling": "Used the printed improvement c = -0.2979 per the harmonisation convention; difference in theta < 0.001."},155 {"issue": f"S4: Table 3 header says 'Time per Email (s)', but the Malicious/Non-Malicious columns (2.8, 1.8, 1.9, 2.4) are implausible as seconds next to Full Corpus 50.5/56.9 s and do not reproduce their printed % savings (e.g. 1-2.8/1.9 = -47.4% vs printed -52.9); likely minutes and/or rounding. Also 1-50.5/56.9 = {100*(1-50.5/56.9):.2f}% vs printed 11.1.",156 "handling": "Only the Full Corpus column is used, with the printed % Time Savings (11.1) as the harmonisation input; sign convention: positive savings means less time with AI, so c = -0.111."},157 {"issue": "S4: only a two-sided p-value (0.471) is printed for the full-corpus time saving; the test underlying it (likely on raw seconds, not logs) is not described in the extract.",158 "handling": "Approximate SE = |theta|/z_(1-p/2) as permitted; study pooled with an approximate SE and flagged."},159 {"issue": "S5: Table 1 notes print the significance key as '** p<0.5' (presumably p<0.05); the row label renders as '\\times TreatPost'.",160 "handling": "Recorded as printed; S5 is not harmonisable regardless."},161 {"issue": "Stray 'p<' / 'p=' lines and footnote numbers in the extracts (rendering artefacts per data/README.md).",162 "handling": "Ignored."},163]164165res = dict(166 task_id="T3-synthesis",167 target_definition="theta = ln(T_AI / T_noAI): natural log of the ratio of mean task-completion time with generative-AI assistance to mean completion time without it, on the same tasks, from a randomised comparison (negative = faster with AI).",168 items=items,169 k_extracted=sum(1 for k in items if items[k]["estimate_as_printed"]),170 k_harmonised=sum(1 for k in items if items[k]["harmonised_estimate"] is not None),171 k_pooled=len(pool),172 fe_estimate=fe, fe_se=fe_se, fe_ci_low=fe - 1.96 * fe_se, fe_ci_high=fe + 1.96 * fe_se,173 re_estimate=re, re_se=re_se, re_ci_low=re_lo, re_ci_high=re_hi,174 q_stat=Q, i2=i2, tau2=tau2, heterogeneity_interpretation=het,175 pooled_significant_5pct=bool(re_lo > 0 or re_hi < 0),176 pooled_pct_change_re=100 * (math.exp(re) - 1),177 feasibility="complete",178 feasibility_reason=None,179 data_issues=data_issues,180 n_input_rows=n_input,181 n_analysed=len(items),182 random_seed=None,183)184with open(os.path.join(OUT, "results.json"), "w") as f:185 json.dump(res, f, indent=2)186187cols = ["estimate_as_printed", "uncertainty_as_printed", "sample_as_printed", "definition", "locator",188 "harmonised_estimate", "harmonised_se", "harmonisable", "poolable", "transformation", "reason_if_not"]189pd.DataFrame([{"study": k, **{c: items[k][c] for c in cols}} for k in items]).to_csv(os.path.join(OUT, "table1.csv"), index=False)190191# ---------------- forest plot ----------------192short = {s_["id"]: s_["short"].split(",")[0] for s_ in studies}193harm = [k for k in items if items[k]["harmonisable"]]194fig, ax = plt.subplots(figsize=(8.5, 4.2))195ypos = list(range(len(harm) + 1, 1, -1))196for yp, k in zip(ypos, harm):197 t = items[k]["harmonised_estimate"]; se = items[k]["harmonised_se"]198 if se is not None:199 ax.errorbar(t, yp, xerr=1.96 * se, fmt="s", color="k", capsize=3)200 lab = f"{t:+.3f} [{t-1.96*se:+.3f}, {t+1.96*se:+.3f}]" + (" (approx. SE)" if k == "S4" else "")201 else:202 ax.plot(t, yp, "o", mfc="white", mec="gray")203 lab = f"{t:+.3f} (no SE; not pooled)"204 ax.text(1.02, yp, lab, transform=ax.get_yaxis_transform(), va="center", fontsize=8)205ax.fill([re_lo, re, re_hi, re], [1, 1.25, 1, 0.75], color="tab:blue")206ax.text(1.02, 1, f"{re:+.3f} [{re_lo:+.3f}, {re_hi:+.3f}]", transform=ax.get_yaxis_transform(), va="center", fontsize=8)207ax.axvline(0, color="gray", ls="--", lw=0.8)208ax.set_yticks(ypos + [1]); ax.set_yticklabels([f"{k} {short[k]}" for k in harm] + [f"RE pooled (k={len(pool)}: {', '.join(pool)})"])209ax.set_xlabel("theta = ln(T_AI / T_noAI) (negative = faster with AI)")210ax.set_title("Harmonised effects of generative-AI access on task-completion time\n(S5 not harmonisable; open circles: no SE available)", fontsize=10)211ax.set_ylim(0.3, len(harm) + 1.7)212plt.tight_layout(); plt.savefig(os.path.join(OUT, "figure1.png"), dpi=150, bbox_inches="tight")213214print(json.dumps({k: v for k, v in res.items() if k not in ("items", "data_issues")}, indent=2))215for k in items:216 print(k, items[k]["harmonised_estimate"], items[k]["harmonised_se"], items[k]["poolable"])217