1"""T3-synthesis: extraction, harmonisation onto theta = ln(T_AI/T_noAI), FE/DL pooling, forest plot.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Reads only data/. Writes out/results.json, out/table1.csv, out/figure1.png. No randomness."""4import json, math, os, re, csv5import numpy as np6from scipy.stats import norm7import matplotlib8matplotlib.use("Agg")9import matplotlib.pyplot as plt1011ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))12D = os.path.join(ROOT, "data"); OUT = os.path.join(ROOT, "out")13Z = 1.961415def lines_of(src):16 with open(os.path.join(D, "sources", src + ".txt"), encoding="utf-8") as fh:17 return fh.read().split("\n")1819def find(src, pattern, start=0):20 """Return (1-based line number, line) of the first line at/after `start` matching regex."""21 L = lines_of(src)22 for i in range(start, len(L)):23 if re.search(pattern, L[i]):24 print(f"[{src} L{i+1}] {L[i].strip()[:220]}")25 return i + 1, L[i]26 raise RuntimeError(f"pattern not found in {src}: {pattern}")2728studies = json.load(open(os.path.join(D, "studies.json"), encoding="utf-8"))29ids = [s["id"] for s in studies]30assert ids == ["S1", "S2", "S3", "S4", "S5"]3132# n_input_rows: wc -l semantics (count of newline characters)33n_input_rows = {"studies.json": len(studies)}34for fn in ["README.md"] + ["sources/" + f for f in sorted(os.listdir(os.path.join(D, "sources")))]:35 with open(os.path.join(D, fn), "rb") as fh:36 n_input_rows[fn] = fh.read().count(b"\n")3738print("=== Source lines relied on ===")39items = {}4041# ---------------- S1 Paradis et al. ----------------42src = "arxiv-2410.12944v3"43ln_tab, row = find(src, r"ExpCon \|")44ln_se, _ = find(src, r"^\(", ln_tab) # next SE line after the ExpCon row45se_line = lines_of(src)[ln_se - 1]46coefs = re.findall(r"-?\d+\.\d+", row.split("ExpCon")[0])47ses = re.findall(r"\((\d+\.\d+)\)", se_line)48b_print, se_print = coefs[1], ses[1] # Model 2 = second column49ln_ci, ci_line = find(src, r"best-fit Model 2")50m = re.search(r"CI = \[(-?\d+\.\d+),\s*(-?\d+\.\d+)\]\s*,\s*p=(\d?\.\d+)", ci_line)51ci_l, ci_u, p1 = m.group(1), m.group(2), m.group(3)52find(src, r"TABLE III")53ln_n, nline = find(src, r"completed the task in 96 min")54b1, se1 = float(b_print), float(se_print)55se1_ci = (float(ci_u) - float(ci_l)) / (2 * Z)56items["S1"] = dict(57 estimate_as_printed=f"{b_print} (ExpCon, Model 2, LogToT; marker '.'); text: beta_1=-0.24, effect of AI = 21%",58 uncertainty_as_printed=f"SE ({se_print}); 95% CI = [{ci_l}, {ci_u}]; p={p1}",59 sample_as_printed="Table III: n = 96 (Section VI-B: N=47 AI, N=46 no-AI)",60 definition="OLS coefficient of the experimental condition (AI vs no-AI, randomised) on the natural log of time on task, controlling for developer-level factors (Model 2)",61 locator=f"{src}; Table III (L{ln_tab}-L{ln_se}), row ExpCon, column (2) 'Dev'; Section VI-C (L{ln_ci}) for CI and p",62 harmonisable=True, harmonised_estimate=b1, harmonised_se=se1, poolable=True,63 transformation=f"log-time coefficient: theta = b = {b_print}; SE(theta) = SE(b) = {se_print} (printed SE used; CI-based SE = ({ci_u} - ({ci_l}))/3.92 = {se1_ci:.4f} used only as sensitivity)",64 reason_if_not=None)6566# ---------------- S2 Becker et al. ----------------67src = "arxiv-2507.09089v2"68ln_a, _ = find(src, r"increases completion time by 19%")69ln_r, rline = find(src, r"implement issues in 19% more time on average")70ln_s, _ = find(src, r"S=\\frac")71ln_c, _ = find(src, r"Developers complete 136 issues")72find(src, r"^Omitted:.*Figure 1")73c2 = float(re.search(r"(\d+)% more time", rline).group(1)) / 10074th2 = math.log(1 + c2)75items["S2"] = dict(76 estimate_as_printed="19% more time on average (AI allowed increases completion time by 19%)",77 uncertainty_as_printed="none printed (confidence interval shown only in Figure 1 / Figure 15, not reproduced in the extract)",78 sample_as_printed="16 developers, 246 tasks; 136 issues AI-allowed, 110 issues AI-disallowed",79 definition="S = E[time with AI allowed]/E[time AI disallowed] - 1, estimated as exp(beta)-1 from OLS of log implementation time on AI indicator and log forecast time (issue-level randomisation)",80 locator=f"{src}; Section 3 Results (L{ln_r}); abstract (L{ln_a}); estimator in Section 2.3 (L{ln_s}) and Appendix D.1-D.3",81 harmonisable=True, harmonised_estimate=th2, harmonised_se=None, poolable=False,82 transformation=f"percentage change: c = +0.19; theta = ln(1 + 0.19) = {th2:.6f} (equals the regression beta since S = exp(beta) - 1)",83 reason_if_not="Not poolable: no SE, CI or p-value is printed in the extract; the confidence interval appears only in a figure that is not reproduced, so SE(theta) cannot be obtained.")8485# ---------------- S3 Bono & Xu ----------------86src = "arxiv-2411.01067v2"87ln_h, _ = find(src, r"\| Overall \| Treatment \| Control \| Improvement")88ln_row, row3 = find(src, r"^All Content \|", ln_h)89cells = [c.strip() for c in row3.split("|")]90ov, tr, co, imp = cells[1], cells[2], cells[3], cells[4]91find(src, r"Table 3: Comparison of task duration")92find(src, r"We had 181 subjects")93find(src, r"^Editorial note")94c3 = float(re.match(r"(-?\d+\.\d+)%", imp).group(1)) / 10095th3 = math.log(1 + c3)96items["S3"] = dict(97 estimate_as_printed=f"Treatment {tr} min, Control {co} min (Overall {ov}); Improvement {imp}",98 uncertainty_as_printed="significance stars only (***; per the paper's scheme 0<p<=0.01); no SE or CI printed",99 sample_as_printed="181 subjects completed the task satisfactorily (one subject dropped)",100 definition="Percentage difference in mean task duration (minutes), Copilot (treatment) vs control without Copilot, holding accuracy constant (control duration predicted at treatment accuracy; Appendix B), randomised",101 locator=f"{src}; Section 4.2, Table 3 (L{ln_h}-L{ln_row}), row 'Aggregate Results / All Content', columns Treatment, Control, Improvement",102 harmonisable=True, harmonised_estimate=th3, harmonised_se=None, poolable=False,103 transformation=f"percentage change: c = {c3}; theta = ln(1 + ({c3})) = {th3:.6f} (ratio of displayed means ln({tr}/{co}) = {math.log(float(tr)/float(co)):.6f}, sensitivity only)",104 reason_if_not="Not poolable: uncertainty is printed only as significance stars (p<=0.01); SE(theta) cannot be recovered from a threshold.")105106# ---------------- S4 Bono ----------------107src = "arxiv-2511.13860v1"108ln_t3, _ = find(src, r"^Table 3: Comparison of Email Processing Times")109ln_aw, aw = find(src, r"^Aware Time per Email", ln_t3)110ln_co, cw = find(src, r"^Control Time per Email", ln_t3)111ln_ts, ts = find(src, r"^% Time Savings", ln_t3)112ln_p, pv = find(src, r"^p-value", ln_t3)113find(src, r"Control \( N\{=\}52")114find(src, r"Aware \( N\{=\}56")115last = lambda s: s.split("|")[-1].strip()116aw4, co4, sav4, p4 = last(aw), last(cw), last(ts), last(pv)117c4 = -float(sav4) / 100118th4 = math.log(1 + c4)119z4 = norm.ppf(1 - float(p4) / 2)120se4 = abs(th4) / z4121items["S4"] = dict(122 estimate_as_printed=f"Aware {aw4}, Control {co4} (row labels: Time per Email (s)); % Time Savings {sav4}",123 uncertainty_as_printed=f"p-value {p4}",124 sample_as_printed="Aware N=56, Control N=52 (of 167 analysts in three arms; 13 low-scoring responses filtered)",125 definition="Percentage time saving per processed email (Full Corpus, whole 25-email queue without resolve-benign protocol), Aware arm (agent verdicts, rationale, agent-ordered queue) vs Control (random order, no AI), randomised",126 locator=f"{src}; Section 3.3, Table 3 (L{ln_t3}-L{ln_p}), column 'Full Corpus', rows Aware/Control Time per Email, % Time Savings, p-value",127 harmonisable=True, harmonised_estimate=th4, harmonised_se=se4, poolable=True,128 transformation=f"percentage time saving {sav4}% -> c = {c4}; theta = ln(1 + ({c4})) = {th4:.6f}; APPROXIMATE SE from two-sided p: SE = |theta|/z_(1-{p4}/2) = {abs(th4):.6f}/{z4:.6f} = {se4:.6f} (ratio of displayed means ln({aw4}/{co4}) = {math.log(float(aw4)/float(co4)):.6f}, sensitivity only)",129 reason_if_not=None)130131# ---------------- S5 Kreitmeir & Raschky ----------------132src = "arxiv-2304.09339v1"133ln_tp, tp = find(src, r"TreatPost \|")134ln_tse, tse = find(src, r"^\| \(", ln_tp)135ln_N, nl = find(src, r"^N \|", ln_tp)136find(src, r"aggregate Output measure")137b5 = [c.strip() for c in tp.split("|")][3]138s5 = [c.strip() for c in tse.split("|")][3]139n5 = [c.strip() for c in nl.split("|")][3]140items["S5"] = dict(141 estimate_as_printed=f"{b5} (Treat x Post, Output, panel A, column (3))",142 uncertainty_as_printed=f"SE {s5} (clustered at user level)",143 sample_as_printed=f"N = {n5} (user-day observations, users with affiliation)",144 definition="Difference-in-differences coefficient (Italy vs Austria/France, post vs pre ChatGPT ban) on an aggregate count of GitHub Output events (Push, Pull Requests, comments, Create, Issues)",145 locator=f"{src}; Section 4, Table 1 (L{ln_tp}-L{ln_N}), panel A 'Pre 27/28.03 - Post 03/04.04', row Treat x Post, column (3) Output (users with affiliation)",146 harmonisable=False, harmonised_estimate=None, harmonised_se=None, poolable=False,147 transformation="none: no transformation maps an effect on output counts onto ln(T_AI/T_noAI)",148 reason_if_not="Not harmonisable: the effect is on output counts (GitHub events) around the withdrawal of ChatGPT access in a non-randomised difference-in-differences design, not a comparison of task-completion times with and without AI on the same tasks.")149150# ---------------- Pooling ----------------151def pool(est, se):152 est, se = np.asarray(est, float), np.asarray(se, float)153 w = 1 / se**2154 fe = float(np.sum(w * est) / np.sum(w)); fe_se = float(math.sqrt(1 / np.sum(w)))155 q = float(np.sum(w * (est - fe) ** 2)); df = len(est) - 1156 C = float(np.sum(w) - np.sum(w**2) / np.sum(w))157 tau2 = max(0.0, (q - df) / C) if C > 0 else 0.0158 i2 = max(0.0, (q - df) / q) * 100 if q > 0 else 0.0159 wr = 1 / (se**2 + tau2)160 re_ = float(np.sum(wr * est) / np.sum(wr)); re_se = float(math.sqrt(1 / np.sum(wr)))161 return dict(fe=fe, fe_se=fe_se, q=q, df=df, tau2=tau2, i2=i2, re=re_, re_se=re_se)162163pool_ids = [k for k in ids if items[k]["poolable"]]164P = pool([items[k]["harmonised_estimate"] for k in pool_ids], [items[k]["harmonised_se"] for k in pool_ids])165interp = "low" if P["i2"] < 25 else ("moderate" if P["i2"] < 50 else "substantial")166re_lo, re_hi = P["re"] - Z * P["re_se"], P["re"] + Z * P["re_se"]167168# sensitivity (not headline)169S_ci = pool([b1, th4], [se1_ci, se4])170th4_ratio = math.log(float(aw4) / float(co4)); se4_ratio = abs(th4_ratio) / z4171S_ratio = pool([b1, th4_ratio], [se1, se4_ratio])172harm = [items[k]["harmonised_estimate"] for k in ids if items[k]["harmonisable"]]173174data_issues = [175 {"issue": "S1 (Paradis): Table III states n = 96, while Section VI-B reports N=47 (AI) + N=46 (no-AI) = 93 completed tasks.", "handling": "Both sample statements recorded in sample_as_printed; coefficient and SE taken from Table III as designated; no effect on theta."},176 {"issue": "S1 (Paradis): Section VI-B says the AI group (Mean LogToT=4.6) was faster than the no-AI group (Mean LogToT=4.3), which contradicts the raw means (96 vs 114 min); the log means appear swapped.", "handling": "Not used; harmonisation relies on the Model 2 regression coefficient designated in studies.json."},177 {"issue": "S1 (Paradis): printed SE 0.14 and printed 95% CI [-0.51, 0.03] imply slightly different SEs (0.14 vs 0.1378); '21% faster' denotes a 21% reduction in time (1-exp(beta)).", "handling": "Printed SE used (convention for a coefficient with SE); CI-based SE reported as sensitivity_ci_se_s1_* keys."},178 {"issue": "S2 (Becker): the confidence interval for the 19% estimate is shown only in Figure 1/Figure 15, which are not reproduced in the extract.", "handling": "No SE imputed; S2 harmonised (theta = ln 1.19) but not poolable."},179 {"issue": "S3 (Bono & Xu): uncertainty only as significance stars with a non-standard scheme (* 0.05<p<=0.1, ** 0.01<p<=0.05, *** p<=0.01); displayed minutes give 44.29/63.10-1 = -29.81% vs printed -29.79%; control subjects could use other AI tools (25% did).", "handling": "Printed -29.79% used for theta; S3 not poolable; contamination noted as a limitation."},180 {"issue": "S4 (Bono): printed % Time Savings 11.1 does not equal 1-50.5/56.9 = 11.25% (Malicious and Non-Malicious columns also do not reproduce from displayed times); row labels say seconds per email, but Full Corpus values (50.5, 56.9) are inconsistent with the per-email Malicious/Non-Malicious values (2.8/1.9, 1.8/2.4), so the unit of the Full Corpus column is ambiguous.", "handling": "Printed % saving (11.1) used for theta (unit-free); ratio of displayed means used as sensitivity (sensitivity_ratio_s4_* keys); SE recovered approximately from p = 0.471."},181 {"issue": "S4 (Bono): the Aware arm differs from Control by both agent verdicts/rationales and agent-prioritised queue ordering; only a p-value is printed.", "handling": "Treated as AI vs no-AI; SE flagged as approximate."},182 {"issue": "S5 (Kreitmeir & Raschky): table notes print '** p<0.5' (presumably 0.05); row label rendered as '\\times TreatPost'; date line under the author block is a rendering artefact; whether Output is logged is ambiguous in Section 3.1.", "handling": "Values copied verbatim; S5 not harmonisable regardless of these issues."},183 {"issue": "Stray 'p<', 'p=' and footnote-number lines in the extracts.", "handling": "Ignored as rendering artefacts per data/README.md."},184]185186res = {187 "task_id": "T3-synthesis",188 "target_definition": "theta = ln(T_AI / T_noAI): natural log of the ratio of mean task-completion time with access to a generative-AI assistant to mean completion time without it, for the same tasks, from a randomised comparison; negative values mean faster with AI.",189 "items": items,190 "k_extracted": sum(1 for k in ids if items[k]["estimate_as_printed"]),191 "k_harmonised": sum(1 for k in ids if items[k]["harmonised_estimate"] is not None),192 "k_pooled": len(pool_ids),193 "pooled_study_ids": pool_ids,194 "fe_estimate": P["fe"], "fe_se": P["fe_se"], "fe_ci_low": P["fe"] - Z * P["fe_se"], "fe_ci_high": P["fe"] + Z * P["fe_se"],195 "re_estimate": P["re"], "re_se": P["re_se"], "re_ci_low": re_lo, "re_ci_high": re_hi,196 "q_stat": P["q"], "q_df": P["df"], "i2": P["i2"], "tau2": P["tau2"],197 "heterogeneity_interpretation": interp,198 "pooled_significant_5pct": bool(re_lo > 0 or re_hi < 0),199 "pooled_pct_change_re": 100 * (math.exp(P["re"]) - 1),200 "pooled_pct_change_re_ci_low": 100 * (math.exp(re_lo) - 1),201 "pooled_pct_change_re_ci_high": 100 * (math.exp(re_hi) - 1),202 "baseline_unweighted_mean_theta_harmonised": float(np.mean(harm)),203 "baseline_unweighted_mean_theta_pooled": float(np.mean([items[k]["harmonised_estimate"] for k in pool_ids])),204 "sensitivity_ci_se_s1_re_estimate": S_ci["re"], "sensitivity_ci_se_s1_re_se": S_ci["re_se"],205 "sensitivity_ratio_s4_theta": th4_ratio, "sensitivity_ratio_s4_re_estimate": S_ratio["re"], "sensitivity_ratio_s4_re_se": S_ratio["re_se"],206 "feasibility": "complete",207 "feasibility_reason": None,208 "data_issues": data_issues,209 "n_input_rows": n_input_rows,210 "n_analysed": len(ids),211 "random_seed": None,212}213os.makedirs(OUT, exist_ok=True)214with open(os.path.join(OUT, "results.json"), "w") as fh:215 json.dump(res, fh, indent=1)216217cols = ["study", "estimate_as_printed", "uncertainty_as_printed", "sample_as_printed", "definition", "locator",218 "harmonised_estimate", "harmonised_se", "harmonisable", "poolable", "transformation", "reason_if_not"]219with open(os.path.join(OUT, "table1.csv"), "w", newline="") as fh:220 w = csv.writer(fh); w.writerow(cols)221 for k in ids:222 it = items[k]223 w.writerow([k] + [("" if it[c] is None else it[c]) for c in cols[1:]])224225# ---------------- Forest plot ----------------226short = {s["id"]: s["short"].split(",")[0] for s in studies}227hk = [k for k in ids if items[k]["harmonisable"]]228fig, ax = plt.subplots(figsize=(8, 4.2))229ys = list(range(len(hk) + 1, 1, -1))230for y, k in zip(ys, hk):231 t, s = items[k]["harmonised_estimate"], items[k]["harmonised_se"]232 if s is not None:233 ax.errorbar(t, y, xerr=Z * s, fmt="s", color="k", capsize=3)234 lab = f"{t:.3f} [{t-Z*s:.3f}, {t+Z*s:.3f}]" + (" (approx. SE)" if k == "S4" else "")235 else:236 ax.plot(t, y, "o", mfc="white", mec="k")237 lab = f"{t:.3f} (no SE; not pooled)"238 ax.text(0.45, y, lab, va="center", fontsize=8)239ax.fill([re_lo, P["re"], re_hi, P["re"]], [1, 1.25, 1, 0.75], color="tab:blue")240ax.text(0.45, 1, f"RE (k={len(pool_ids)}): {P['re']:.3f} [{re_lo:.3f}, {re_hi:.3f}]", va="center", fontsize=8)241ax.axvline(0, color="grey", ls="--", lw=0.8)242ax.set_yticks(ys + [1]); ax.set_yticklabels([f"{k} {short[k]}" for k in hk] + ["Pooled DL random effects"])243ax.set_xlim(-0.9, 1.3); ax.set_xlabel("theta = ln(T_AI / T_noAI) (negative = faster with AI)")244ax.set_title("Forest plot: harmonised effects of AI access on task-completion time\n(S5 not harmonisable; S2, S3 shown without intervals)", fontsize=9)245fig.tight_layout(); fig.savefig(os.path.join(OUT, "figure1.png"), dpi=150); plt.close(fig)246247print("=== Results ===")248for k in ids:249 print(k, "theta =", items[k]["harmonised_estimate"], "SE =", items[k]["harmonised_se"], "poolable =", items[k]["poolable"])250print(f"FE {res['fe_estimate']:.4f} (SE {res['fe_se']:.4f}); RE {res['re_estimate']:.4f} (SE {res['re_se']:.4f}) "251 f"[{re_lo:.4f}, {re_hi:.4f}]; Q={P['q']:.4f}, I2={P['i2']:.2f}%, tau2={P['tau2']:.4f}; pct change RE {res['pooled_pct_change_re']:.2f}%")252