1"""T3-synthesis: extraction, harmonisation onto theta = ln(T_AI/T_noAI), FE/DL pooling, forest plot.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Reads only data/. Deterministic (no randomness; random_seed = null)."""4import json, math, os, re, csv5import numpy as np6from scipy.stats import norm7import matplotlib8matplotlib.use("Agg")9import matplotlib.pyplot as plt1011ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))12DATA = os.path.join(ROOT, "data")13OUT = os.path.join(ROOT, "out")14Z = 1.961516def lines_of(src):17 with open(os.path.join(DATA, "sources", src + ".txt"), encoding="utf-8") as fh:18 return fh.read().split("\n")1920def find(src, pattern, nth=0):21 """Return (line_no, line) of the nth line matching pattern; print it."""22 L = lines_of(src)23 hits = [(i + 1, l) for i, l in enumerate(L) if re.search(pattern, l)]24 if len(hits) <= nth:25 raise RuntimeError(f"pattern {pattern!r} not found in {src}")26 ln, line = hits[nth]27 show = line if len(line) < 400 else "..." + line[max(0, re.search(pattern, line).start() - 150): re.search(pattern, line).end() + 150] + "..."28 print(f"[{src} L{ln}] {show}")29 return ln, line3031def wc_l(path):32 with open(path, "rb") as fh:33 return fh.read().count(b"\n")3435studies = json.load(open(os.path.join(DATA, "studies.json")))36assert [s["id"] for s in studies] == ["S1", "S2", "S3", "S4", "S5"]37print("== Source lines relied upon ==")3839items = {}40data_issues = []4142# ---------------- S1 Paradis et al. 2024 ----------------43src = "arxiv-2410.12944v3"44ln_c, l_c = find(src, r"ExpCon \|")45ln_s = ln_c + 246se_line = lines_of(src)[ln_s - 1]; print(f"[{src} L{ln_s}] {se_line}")47coefs = re.findall(r"-?\d+\.\d+", l_c); ses = re.findall(r"\((\d+\.\d+)\)", se_line)48b1, se1 = float(coefs[1]), float(ses[1]) # Model 2 column49assert (b1, se1) == (-0.24, 0.14)50ln_ci, l_ci = find(src, r"best-fit Model 2")51m = re.search(r"\[(-?\d+\.\d+),\s*(-?\d+\.\d+)\]\s*,\s*p=(\d*\.\d+)", l_ci)52ci_l, ci_u, p1 = float(m.group(1)), float(m.group(2)), float(m.group(3))53ln_n, l_n = find(src, r"TABLE III")54ln_m, l_m = find(src, r"Mean LogToT")55se1_ci = (ci_u - ci_l) / (2 * Z)56items["S1"] = dict(57 estimate_as_printed="ExpCon coefficient, Model 2 (Dev): -0.24 (marked '.'); quoted in VI-C as beta_1=-0.24, i.e. AI effect (1-exp(beta_1))*100 = 21%",58 uncertainty_as_printed="SE (0.14); 95% CI = [-0.51, 0.03]; p=0.086 (NS)",59 sample_as_printed="n = 96 (Table III caption); VI-B: N=47 (AI), N=46 (no-AI)",60 definition="OLS coefficient on experimental condition (AI vs no-AI) with developer-level covariates; dependent variable = natural log of time on task (LogToT); randomised enterprise RCT, one complex coding task",61 locator=f"{src}, Table III (L{ln_n}), row ExpCon (L{ln_c}) and SE row (L{ln_s}), column (2) 'Dev' = Model 2; CI and p-value in Section VI-C (L{ln_ci})",62 harmonisable=True, harmonised_estimate=b1, harmonised_se=se1, poolable=True,63 transformation=f"log-time coefficient: theta = b = -0.24; SE(theta) = SE(b) = 0.14 (CI-implied alternative (0.03-(-0.51))/(2*1.96) = {se1_ci:.4f})",64 reason_if_not=None)65data_issues.append({"issue": f"S1: Section VI-B (L{ln_m}) prints Mean LogToT = 4.6 for the AI group and 4.3 for the no-AI group while stating the AI group was faster (96 vs 114 min; ln 96 = 4.56, ln 114 = 4.74): internally inconsistent.",66 "handling": "Mean LogToT values not used; theta taken from the Table III Model 2 coefficient."})67data_issues.append({"issue": "S1: Section VI-B reports N=47+46=93 developers while Table III states n = 96.",68 "handling": "Both printed as is in sample_as_printed; not needed for theta or SE."})69data_issues.append({"issue": f"S1: the CI [-0.51, 0.03] implies SE = {se1_ci:.4f} whereas Table III prints SE = 0.14 (rounding).",70 "handling": "Printed SE(b) = 0.14 used (first harmonisation convention; reproduces p = 0.086 under a normal approximation); sensitivity with the CI-implied SE reported as sens_* keys."})7172# ---------------- S2 Becker et al. 2025 ----------------73src = "arxiv-2507.09089v2"74ln_a, l_a = find(src, r"increases completion time by 19%")75ln_r, l_r = find(src, r"19% more time on average")76ln_d, l_d = find(src, r"Developers complete 136 issues")77ln_f, l_f = find(src, r"Figure 1, which displays the confidence interval")78c2 = 0.1979items["S2"] = dict(80 estimate_as_printed="+19% (\"allowing AI actually increases completion time by 19%\"; \"they implement issues in 19% more time on average\")",81 uncertainty_as_printed="none printed in the extract (95% CI shown only in Figure 1/Figure 15, not reproduced)",82 sample_as_printed="16 developers, 246 tasks (136 issues AI-allowed, 110 AI-disallowed)",83 definition="S = E[time with AI allowed]/E[time with AI disallowed] - 1, estimated as exp(beta)-1 from OLS of log implementation time on AI indicator and log forecast time (issue-level randomisation)",84 locator=f"{src}, Abstract (L{ln_a}); Section 3 Results (L{ln_r}); estimator Sections 2.3 and D.1-D.3",85 harmonisable=True, harmonised_estimate=math.log(1 + c2), harmonised_se=None, poolable=False,86 transformation="percentage change c = +0.19: theta = ln(1 + 0.19) = ln(1.19)",87 reason_if_not="Not poolable: no SE, CI or p-value for the 19% estimate is printed in the extract; its confidence interval appears only in figures that are not reproduced.")8889# ---------------- S3 Bono & Xu 2024 ----------------90src = "arxiv-2411.01067v2"91ln_t, l_t = find(src, r"^All Content \| 53\.84")92ln_cap, l_cap = find(src, r"^Table 3: Comparison of task duration")93ln_ed, l_ed = find(src, r"^Editorial note: the paper marks significance")94cells = [c.strip() for c in l_t.split("|")]95overall, treat3, ctrl3 = float(cells[1]), float(cells[2]), float(cells[3])96imp = cells[4]; c3 = float(re.match(r"(-?\d+\.\d+)%", imp).group(1)) / 10097assert (treat3, ctrl3, c3) == (44.29, 63.10, -0.2979)98items["S3"] = dict(99 estimate_as_printed="All Content: Overall 53.84 | Treatment 44.29 | Control 63.10 (minutes) | Improvement -29.79%***",100 uncertainty_as_printed="significance stars only: *** (0<p<=0.01); no SE or CI printed",101 sample_as_printed="181 subjects completed the task satisfactorily (one further subject dropped)",102 definition="Difference in mean task duration (minutes), Copilot vs control, holding accuracy constant (control duration predicted at the treatment group's accuracy, Appendix B); randomised assignment",103 locator=f"{src}, Section 4.2, Table 3 (L{ln_cap}), 'Aggregate Results' block, row 'All Content' (L{ln_t}), columns Treatment/Control/Improvement",104 harmonisable=True, harmonised_estimate=math.log(1 + c3), harmonised_se=None, poolable=False,105 transformation=f"percentage change c = -0.2979: theta = ln(1 - 0.2979) = ln(0.7021) (ratio of printed durations 44.29/63.10 would give {math.log(treat3/ctrl3):.5f})",106 reason_if_not="Not poolable: uncertainty is reported by significance stars only (***), so SE(theta) cannot be obtained.")107data_issues.append({"issue": f"S3: printed durations give 44.29/63.10 - 1 = {100*(treat3/ctrl3-1):.2f}% whereas the printed improvement is -29.79%.",108 "handling": "Printed improvement c = -0.2979 used (theta = ln(1+c)); the ratio-based value differs by < 0.001 on the theta scale; S3 is not pooled anyway."})109110# ---------------- S4 Bono 2025 ----------------111src = "arxiv-2511.13860v1"112ln_h, l_h = find(src, r"^Metric \| Malicious")113ln_aw, l_aw = find(src, r"^Aware Time per Email")114ln_co, l_co = find(src, r"^Control Time per Email")115ln_ts, l_ts = find(src, r"^% Time Savings")116ln_p, l_p = find(src, r"^p-value \|")117ln_arms, l_arms = find(src, r"Aware \( N\{=\}56")118last = lambda l: float(l.split("|")[-1].strip())119aware4, ctrl4, sav4, p4 = last(l_aw), last(l_co), last(l_ts), last(l_p)120assert (aware4, ctrl4, sav4, p4) == (50.5, 56.9, 11.1, 0.471)121c4 = -sav4 / 100122th4 = math.log(1 + c4)123z4 = norm.ppf(1 - p4 / 2)124se4 = abs(th4) / z4125items["S4"] = dict(126 estimate_as_printed="Full Corpus: Aware Time per Email 50.5 s | Control Time per Email 56.9 s | % Time Savings 11.1",127 uncertainty_as_printed="p-value 0.471",128 sample_as_printed="Aware N=56, Control N=52 (Blind N=59; 167 analysts in total, 13 low-scoring responses filtered); 25-email queue per subject",129 definition="Mean time per email (seconds), Aware arm (agent verdicts, rationale and agent-ordered queue) vs Control arm (no AI), full queue; % time savings = reduction relative to Control; randomised three-arm RCT",130 locator=f"{src}, Section 3.3, Table 3 (L{ln_h}), rows Aware/Control Time per Email, % Time Savings, p-value (L{ln_aw}-L{ln_p}), column 'Full Corpus'",131 harmonisable=True, harmonised_estimate=th4, harmonised_se=se4, poolable=True,132 transformation=f"time saving 11.1% -> c = -0.111: theta = ln(1 - 0.111) = ln(0.889) = {th4:.5f}; approximate SE from two-sided p: SE = |theta| / z_(1-0.471/2) = {abs(th4):.5f} / {z4:.5f} = {se4:.5f} (APPROXIMATE SE)",133 reason_if_not=None)134data_issues.append({"issue": f"S4: printed Full Corpus times give 1 - 50.5/56.9 = {100*(1-aware4/ctrl4):.2f}% savings, not the printed 11.1%; the Malicious/Non-Malicious columns (2.8, 1.8, 1.9, 2.4, labelled seconds) do not match the full-corpus magnitude (50.5/56.9 s) and their implied savings differ from the printed -52.9/23.6 (ambiguous units/rounding).",135 "handling": "Only the Full Corpus column is used, with the printed % time savings (11.1) and p-value; the ambiguous columns are not used."})136data_issues.append({"issue": "S4: only a p-value is printed; the SE is recovered as |theta|/z_(1-p/2), which assumes a normal test statistic on the log-ratio scale (the paper's test may be on another scale).",137 "handling": "S4 pooled with an explicitly approximate SE, as allowed by the task conventions; flagged in report."})138139# ---------------- S5 Kreitmeir & Raschky 2023 ----------------140src = "arxiv-2304.09339v1"141ln_hd, l_hd = find(src, r"^\| Events \| Release \| Output")142ln_tp, l_tp = find(src, r"TreatPost \| 0\.115")143ln_tps = ln_tp + 2; l_tps = lines_of(src)[ln_tps - 1]; print(f"[{src} L{ln_tps}] {l_tps}")144ln_N, l_N = find(src, r"^N \| 18332")145ln_leg, l_leg = find(src, r"\*\* p<0\.5")146c5 = [c.strip() for c in l_tp.split("|")][3]; s5 = [c.strip() for c in l_tps.split("|")][3]; n5 = [c.strip() for c in l_N.split("|")][3]147assert (c5, s5, n5) == ("-0.278", "(0.295)", "18332")148items["S5"] = dict(149 estimate_as_printed="Treat x Post (Output, column 3, panel A, users with affiliation): -0.278",150 uncertainty_as_printed="SE (0.295), no significance stars",151 sample_as_printed="N = 18332 (user-day observations)",152 definition="Difference-in-differences coefficient on an aggregate GitHub Output count measure (Push, Pull Requests, PR comments, commit comments, Create, Issues) for Italian vs Austrian/French users, pre (27/28.03.2023) vs post (03/04.04.2023) the ChatGPT ban; user FE, user-clustered SE",153 locator=f"{src}, Section 4, Table 1 (header L{ln_hd}), panel A, row Treat x Post (L{ln_tp}) and SE row (L{ln_tps}), column (3) Output; N row (L{ln_N})",154 harmonisable=False, harmonised_estimate=None, harmonised_se=None, poolable=False,155 transformation="none applied (not harmonisable)",156 reason_if_not="Not harmonisable: the printed effect is on output counts (activity volume) around a withdrawal of access in an observational DiD, not a comparison of task-completion times with and without AI on the same tasks.")157data_issues.append({"issue": f"S5: Table 1 notes print the significance legend as '** p<0.5' (L{ln_leg}), presumably a typo for p<0.05.",158 "handling": "Irrelevant to the analysis (S5 not harmonisable); recorded."})159160# ---------------- pooling ----------------161def pool(th, se):162 th, se = np.asarray(th, float), np.asarray(se, float)163 w = 1 / se**2164 fe = float(np.sum(w * th) / np.sum(w)); fe_se = float(math.sqrt(1 / np.sum(w)))165 k = len(th); df = k - 1166 Q = float(np.sum(w * (th - fe) ** 2))167 C = float(np.sum(w) - np.sum(w**2) / np.sum(w))168 tau2 = max(0.0, (Q - df) / C) if C > 0 else 0.0169 i2 = max(0.0, (Q - df) / Q) * 100 if Q > 0 else 0.0170 wr = 1 / (se**2 + tau2)171 re_ = float(np.sum(wr * th) / np.sum(wr)); re_se = float(math.sqrt(1 / np.sum(wr)))172 return dict(fe=fe, fe_se=fe_se, re=re_, re_se=re_se, Q=Q, df=df, C=C, tau2=tau2, i2=i2,173 w_fe=(w / w.sum()).tolist(), w_re=(wr / wr.sum()).tolist())174175pool_ids = [k for k in items if items[k]["poolable"]]176P = pool([items[k]["harmonised_estimate"] for k in pool_ids], [items[k]["harmonised_se"] for k in pool_ids])177# sensitivity: S1 with CI-implied SE178Ps = pool([items[k]["harmonised_estimate"] for k in pool_ids],179 [se1_ci if k == "S1" else items[k]["harmonised_se"] for k in pool_ids])180harm_ids = [k for k in items if items[k]["harmonised_estimate"] is not None]181i2 = P["i2"]182het = "low" if i2 < 25 else ("moderate" if i2 < 50 else "substantial")183re_lo, re_hi = P["re"] - Z * P["re_se"], P["re"] + Z * P["re_se"]184185n_input_rows = {"studies.json": len(studies)}186for s in studies:187 fn = s["source"] + ".txt"188 n_input_rows[fn] = wc_l(os.path.join(DATA, "sources", fn))189190results = {191 "task_id": "T3-synthesis",192 "target_definition": "theta = ln(T_AI / T_noAI): natural log of the ratio of mean task-completion time with generative-AI assistance to mean completion time without it, for the same tasks, from a randomised comparison (negative = faster with AI).",193 "items": items,194 "k_extracted": sum(1 for k in items if items[k]["estimate_as_printed"]),195 "k_harmonised": len(harm_ids),196 "k_pooled": len(pool_ids),197 "pooled_ids": pool_ids,198 "fe_estimate": P["fe"], "fe_se": P["fe_se"],199 "fe_ci_low": P["fe"] - Z * P["fe_se"], "fe_ci_high": P["fe"] + Z * P["fe_se"],200 "re_estimate": P["re"], "re_se": P["re_se"], "re_ci_low": re_lo, "re_ci_high": re_hi,201 "q_stat": P["Q"], "q_df": P["df"], "q_pvalue": float(1 - __import__("scipy.stats").stats.chi2.cdf(P["Q"], P["df"])),202 "i2": i2, "tau2": P["tau2"],203 "heterogeneity_interpretation": het,204 "pooled_significant_5pct": bool(re_lo > 0 or re_hi < 0),205 "pooled_pct_change_re": 100 * (math.exp(P["re"]) - 1),206 "pooled_pct_change_re_ci": [100 * (math.exp(re_lo) - 1), 100 * (math.exp(re_hi) - 1)],207 "weights_fe": dict(zip(pool_ids, P["w_fe"])),208 "weights_re": dict(zip(pool_ids, P["w_re"])),209 "baseline_unweighted_mean_theta_harmonised": float(np.mean([items[k]["harmonised_estimate"] for k in harm_ids])),210 "baseline_unweighted_mean_theta_poolable": float(np.mean([items[k]["harmonised_estimate"] for k in pool_ids])),211 "baseline_vote_count_faster_with_ai": sum(1 for k in harm_ids if items[k]["harmonised_estimate"] < 0),212 "sens_re_estimate_s1_ci_se": Ps["re"], "sens_re_se_s1_ci_se": Ps["re_se"],213 "sens_re_ci_low_s1_ci_se": Ps["re"] - Z * Ps["re_se"], "sens_re_ci_high_s1_ci_se": Ps["re"] + Z * Ps["re_se"],214 "feasibility": "complete",215 "feasibility_reason": None,216 "data_issues": data_issues,217 "n_input_rows": n_input_rows,218 "n_analysed": len(items),219 "random_seed": None,220}221222with open(os.path.join(OUT, "results.json"), "w") as fh:223 json.dump(results, fh, indent=1)224225cols = ["study", "estimate_as_printed", "uncertainty_as_printed", "sample_as_printed", "definition", "locator",226 "harmonised_estimate", "harmonised_se", "harmonisable", "poolable", "transformation", "reason_if_not"]227with open(os.path.join(OUT, "table1.csv"), "w", newline="") as fh:228 w = csv.writer(fh); w.writerow(cols)229 for k, it in items.items():230 w.writerow([k] + [("" if it[c] is None else it[c]) for c in cols[1:]])231232# ---------------- forest plot ----------------233labels = {s["id"]: s["short"].split(",")[0] for s in studies}234fig, ax = plt.subplots(figsize=(8, 4.2))235rows = harm_ids[::-1]236for y, k in enumerate(rows, start=1):237 th, se = items[k]["harmonised_estimate"], items[k]["harmonised_se"]238 if se is not None:239 ax.errorbar(th, y, xerr=Z * se, fmt="s", color="black", capsize=3)240 txt = f"{th:+.3f} [{th-Z*se:+.3f}, {th+Z*se:+.3f}]" + (" (approx. SE)" if k == "S4" else "")241 else:242 ax.plot(th, y, "o", mfc="white", mec="grey")243 txt = f"{th:+.3f} (no SE: not pooled)"244 ax.text(0.62, y, txt, va="center", fontsize=8)245ax.errorbar(P["re"], 0, xerr=Z * P["re_se"], fmt="D", color="tab:red", capsize=3)246ax.text(0.62, 0, f"{P['re']:+.3f} [{re_lo:+.3f}, {re_hi:+.3f}]", va="center", fontsize=8, color="tab:red")247ax.set_yticks(range(0, len(rows) + 1))248ax.set_yticklabels([f"RE pooled (k={len(pool_ids)}: {', '.join(pool_ids)})"] + [f"{k} {labels[k]}" for k in rows])249ax.axvline(0, color="grey", ls="--", lw=0.8)250ax.set_xlim(-0.75, 1.25)251ax.set_xlabel("theta = ln(T_AI / T_noAI) (negative = faster with AI)")252ax.set_title("Figure 1. Harmonised effects of AI access on task-completion time (95% CI)", fontsize=10)253fig.tight_layout()254fig.savefig(os.path.join(OUT, "figure1.png"), dpi=150)255256print("== Results ==")257for k in items:258 it = items[k]259 print(k, "theta=", it["harmonised_estimate"], "se=", it["harmonised_se"], "poolable=", it["poolable"])260print(f"FE {results['fe_estimate']:.4f} (SE {results['fe_se']:.4f}) [{results['fe_ci_low']:.4f}, {results['fe_ci_high']:.4f}]")261print(f"RE {results['re_estimate']:.4f} (SE {results['re_se']:.4f}) [{re_lo:.4f}, {re_hi:.4f}]; Q={P['Q']:.4f} I2={i2:.2f} tau2={P['tau2']:.4f} ({het})")262print(f"pooled % change (RE) = {results['pooled_pct_change_re']:.2f}%; significant at 5%: {results['pooled_significant_5pct']}")263