1"""T3-synthesis: extraction, harmonisation onto theta = ln(T_AI/T_noAI), IV fixed-effect and2DerSimonian-Laird pooling, forest plot. Reads only data/; writes out/results.json, out/table1.csv,3out/figure1.png. Deterministic; no randomness (random_seed = null).4Run from workspace root: ~/wp29/venv/bin/python out/analysis.py5"""6import csv7import json8import math9import os10import re1112import matplotlib13matplotlib.use("Agg")14import matplotlib.pyplot as plt15from scipy.stats import norm1617DATA = "data"18SRC = os.path.join(DATA, "sources")19OUT = "out"20Z = 1.96212223def lines_of(name):24 with open(os.path.join(SRC, name + ".txt"), encoding="utf-8") as fh:25 return fh.read().split("\n")262728def get_line(src, lineno, must_contain):29 """Return 1-based line `lineno` of a source, asserting it contains the expected text; print it."""30 ln = lines_of(src)[lineno - 1]31 for s in must_contain:32 assert s in ln, f"{src}:{lineno} does not contain {s!r}: {ln!r}"33 print(f"[{src} L{lineno}] {ln.strip()}")34 return ln353637def wc_l(path):38 with open(path, "rb") as fh:39 return fh.read().count(b"\n")404142print("=== Source lines relied on ===")43# ---------------- S1 Paradis et al. 2024 ----------------44s1_coef = get_line("arxiv-2410.12944v3", 76, ["ExpCon"])45s1_se = get_line("arxiv-2410.12944v3", 78, ["(0.14)"])46s1_txt = get_line("arxiv-2410.12944v3", 58, ["95\\%CI = [-0.51,0.03]", "p=0.086"])47s1_n = get_line("arxiv-2410.12944v3", 66, ["n = 96"])48s1_n2 = get_line("arxiv-2410.12944v3", 48, ["N=47", "N=46"])49coefs = s1_coef.split("ExpCon")[0].split() # ['-0.25.', '-0.24.', '-0.30*', '-0.14']50ses = re.findall(r"\(([-\d.]+)\)", s1_se)51b1 = float(coefs[1].rstrip(".*")) # Model 252se1 = float(ses[1])53m = re.search(r"95\\%CI = \[([-\d.]+),([-\d.]+)\]", s1_txt)54L1, U1 = float(m.group(1)), float(m.group(2))55p1 = float(re.search(r"p=([\d.]+)", s1_txt).group(1))56se1_from_ci = (U1 - L1) / (2 * Z)5758# ---------------- S2 Becker et al. 2025 ----------------59s2_abs = get_line("arxiv-2507.09089v2", 34, ["increases completion time by 19%"])60s2_res = get_line("arxiv-2507.09089v2", 62, ["19% more time", "136 issues", "110 issues"])61s2_hdr = get_line("arxiv-2507.09089v2", 8, ["Figure 1, which displays the confidence interval"])62c2 = float(re.search(r"by (\d+)%", s2_abs).group(1)) / 100.06364# ---------------- S3 Bono & Xu 2024 ----------------65s3_row = get_line("arxiv-2411.01067v2", 55, ["All Content", "-29.79%***"])66s3_hdr = get_line("arxiv-2411.01067v2", 51, ["Treatment", "Control", "Improvement"])67s3_n = get_line("arxiv-2411.01067v2", 38, ["181 subjects"])68s3_note = get_line("arxiv-2411.01067v2", 10, ["no standard errors or confidence intervals"])69cells3 = [c.strip() for c in s3_row.split("|")]70overall3, t3, ctl3 = float(cells3[1]), float(cells3[2]), float(cells3[3])71c3 = float(cells3[4].replace("%", "").replace("*", "")) / 100.07273# ---------------- S4 Bono 2025 ----------------74s4_hdr = get_line("arxiv-2511.13860v1", 88, ["Full Corpus"])75s4_aw = get_line("arxiv-2511.13860v1", 90, ["Aware Time per Email (s)"])76s4_ct = get_line("arxiv-2511.13860v1", 92, ["Control Time per Email (s)"])77s4_pct = get_line("arxiv-2511.13860v1", 94, ["% Time Savings"])78s4_p = get_line("arxiv-2511.13860v1", 96, ["p-value"])79s4_nc = get_line("arxiv-2511.13860v1", 31, ["Control ( N{=}52 )"])80s4_na = get_line("arxiv-2511.13860v1", 35, ["Aware ( N{=}56 )"])81s4_f = get_line("arxiv-2511.13860v1", 44, ["We filter out 13 subject responses"])82last = lambda ln: ln.split("|")[-1].strip()83aw4, ct4, sav4, p4 = float(last(s4_aw)), float(last(s4_ct)), float(last(s4_pct)), float(last(s4_p))84c4 = -sav4 / 100.0 # "11.1 % time savings" -> change in time c = -0.1118586# ---------------- S5 Kreitmeir & Raschky 2023 ----------------87s5_hd = get_line("arxiv-2304.09339v1", 74, ["Events", "Release", "Output", "PushPull"])88s5_c = get_line("arxiv-2304.09339v1", 80, ["TreatPost"])89s5_se = get_line("arxiv-2304.09339v1", 82, ["(0.295)"])90s5_n = get_line("arxiv-2304.09339v1", 88, ["N | 18332"])91s5_def = get_line("arxiv-2304.09339v1", 42, ["aggregate Output measure"])92b5 = [c.strip() for c in s5_c.split("|")][3]93se5 = [c.strip() for c in s5_se.split("|")][3]94n5 = [c.strip() for c in s5_n.split("|")][3]95assert b5 == "-0.278" and se5 == "(0.295)" and n5 == "18332"9697# ---------------- Harmonisation ----------------98theta1 = b199theta2 = math.log(1 + c2)100theta3 = math.log(1 + c3)101theta4 = math.log(1 + c4)102z4 = norm.ppf(1 - p4 / 2)103se4 = abs(theta4) / z4104105items = {106 "S1": {107 "estimate_as_printed": f"ExpCon coefficient (Model 2) = {coefs[1]} (marker '.' = p<.1) on LogToT; text: beta_1=-0.24, effect of AI = 21% (faster)",108 "uncertainty_as_printed": f"SE ({ses[1]}); 95% CI = [{m.group(1)}, {m.group(2)}]; p=0.086 (NS)",109 "sample_as_printed": "n = 96 (Table III); time-on-task means N=47 (AI) and N=46 (no AI) in Sec. VI-B",110 "definition": "OLS coefficient of the AI experimental-condition indicator on the natural log of time on task, controlling for developer-level covariates (Model 2); randomised, one common enterprise coding task",111 "locator": "arxiv-2410.12944v3, Table III, row ExpCon (+ SE row below), column (2) 'Dev' [extract lines 76-78]; Section VI-C, 95% CI and p-value [line 58]",112 "harmonisable": True,113 "harmonised_estimate": theta1,114 "harmonised_se": se1,115 "poolable": True,116 "transformation": f"theta = b = {b1}; SE(theta) = SE(b) = {se1} (cross-check from printed CI: ({U1} - ({L1}))/(2*1.96) = {se1_from_ci:.4f})",117 "reason_if_not": None,118 },119 "S2": {120 "estimate_as_printed": "+19% (allowing AI increases completion time by 19%; 'implement issues in 19% more time on average')",121 "uncertainty_as_printed": "none printed (the 95% CI is shown only in Figure 1, which is not reproduced in the extract)",122 "sample_as_printed": "16 developers; 246 tasks (136 issues AI-allowed, 110 AI-disallowed)",123 "definition": "S = E[time with AI allowed]/E[time with AI disallowed] - 1, estimated as exp(beta)-1 from a log-linear OLS of total implementation time on the AI indicator and log forecast time (issue-level randomisation)",124 "locator": "arxiv-2507.09089v2, Abstract [line 34] and Section 3 Results opening [line 62]; estimator Sections 2.3 and D.1-D.3",125 "harmonisable": True,126 "harmonised_estimate": theta2,127 "harmonised_se": None,128 "poolable": False,129 "transformation": f"theta = ln(1 + c) = ln(1 + {c2}) = {theta2:.6f}",130 "reason_if_not": "Not poolable: no SE, CI or p-value for the 19% estimate is printed in the extract; the confidence interval appears only in Figure 1, which is not reproduced, so SE(theta) cannot be obtained.",131 },132 "S3": {133 "estimate_as_printed": f"All Content: Overall {cells3[1]} min, Treatment {cells3[2]} min, Control {cells3[3]} min, Improvement {cells3[4]}",134 "uncertainty_as_printed": "significance stars only: *** (defined as 0<p<=0.01); no SE or CI printed",135 "sample_as_printed": "181 subjects completed the task satisfactorily (randomised half to Copilot)",136 "definition": "Percentage difference in mean task duration (minutes), Copilot vs control, holding accuracy constant (control duration predicted at treatment accuracy; Appendix B), all tasks across three IT-admin scenarios",137 "locator": "arxiv-2411.01067v2, Section 4.2, Table 3, row 'Aggregate Results - All Content', columns Treatment/Control/Improvement [line 55]",138 "harmonisable": True,139 "harmonised_estimate": theta3,140 "harmonised_se": None,141 "poolable": False,142 "transformation": f"theta = ln(1 + c) = ln(1 + ({c3})) = {theta3:.6f} (printed ratio check: ln({t3}/{ctl3}) = {math.log(t3/ctl3):.6f})",143 "reason_if_not": "Not poolable: uncertainty is reported only as significance stars (*** = p<=0.01); bootstrap SEs/CIs are not printed, so SE(theta) cannot be obtained.",144 },145 "S4": {146 "estimate_as_printed": f"Full Corpus: Aware Time per Email (s) {last(s4_aw)}, Control Time per Email (s) {last(s4_ct)}, % Time Savings {last(s4_pct)}",147 "uncertainty_as_printed": f"p-value {last(s4_p)}",148 "sample_as_printed": "Control N=52, Aware N=56 (Blind N=59; 167 analysts in total; 13 responses scoring <40% filtered: 3 Aware, 6 Blind, 4 Control)",149 "definition": "Percentage time saving per processed email (mean time per email), Aware arm (agent queue ordering + verdicts/rationales) vs Control (no AI), full 25-email queue",150 "locator": "arxiv-2511.13860v1, Section 3.3, Table 3, column 'Full Corpus', rows Aware/Control time, % Time Savings, p-value [lines 88-96]",151 "harmonisable": True,152 "harmonised_estimate": theta4,153 "harmonised_se": se4,154 "poolable": True,155 "transformation": f"c = -{sav4}/100 = {c4}; theta = ln(1 + c) = ln({1 + c4:.3f}) = {theta4:.6f}; approximate SE = |theta| / z_(1-p/2) = {abs(theta4):.6f} / z_(1-{p4}/2) = {abs(theta4):.6f} / {z4:.6f} = {se4:.6f} (approximate SE from p-value only)",156 "reason_if_not": None,157 },158 "S5": {159 "estimate_as_printed": f"Treat x Post (Output, column 3, panel A) = {b5}",160 "uncertainty_as_printed": f"SE {se5} (clustered at user level); not significant",161 "sample_as_printed": f"N = {n5} (user-day observations, users with affiliation)",162 "definition": "Difference-in-differences coefficient on an aggregate GitHub output count measure (Push, PRs, comments, Create, Issues) for Italian vs Austrian/French users around Italy's ChatGPT ban (pre 27/28.03 vs post 03/04.04.2023)",163 "locator": "arxiv-2304.09339v1, Section 4, Table 1, panel A, row Treat x Post (+ SE row), column (3) Output; N row [lines 80-88]",164 "harmonisable": False,165 "harmonised_estimate": None,166 "harmonised_se": None,167 "poolable": False,168 "transformation": "none (not harmonisable)",169 "reason_if_not": "Not harmonisable: the effect is on output counts (activity volume) around the withdrawal of access, not a comparison of completion times with vs without AI on the same tasks, and it is not a randomised comparison.",170 },171}172173# ---------------- Pooling ----------------174pool = [(k, v["harmonised_estimate"], v["harmonised_se"]) for k, v in items.items() if v["poolable"]]175y = [p[1] for p in pool]176w = [1 / p[2] ** 2 for p in pool]177sw = sum(w)178fe = sum(wi * yi for wi, yi in zip(w, y)) / sw179fe_se = math.sqrt(1 / sw)180Q = sum(wi * (yi - fe) ** 2 for wi, yi in zip(w, y))181df = len(pool) - 1182C = sw - sum(wi ** 2 for wi in w) / sw183tau2 = max(0.0, (Q - df) / C)184i2 = max(0.0, (Q - df) / Q) * 100 if Q > 0 else 0.0185wr = [1 / (p[2] ** 2 + tau2) for p in pool]186re_est = sum(wi * yi for wi, yi in zip(wr, y)) / sum(wr)187re_se = math.sqrt(1 / sum(wr))188re_lo, re_hi = re_est - Z * re_se, re_est + Z * re_se189het = "low" if i2 < 25 else ("moderate" if i2 < 50 else "substantial")190191harm = {k: v["harmonised_estimate"] for k, v in items.items() if v["harmonised_estimate"] is not None}192193n_input_rows = {194 "README.md": wc_l(os.path.join(DATA, "README.md")),195 "studies.json": len(json.load(open(os.path.join(DATA, "studies.json")))),196}197for f in sorted(os.listdir(SRC)):198 n_input_rows[f] = wc_l(os.path.join(SRC, f))199200data_issues = [201 {"issue": "S1 (Paradis): Sec. VI-B prints Mean LogToT = 4.6 for the AI group and 4.3 for the no-AI group, which contradicts the reported faster AI group (96 vs 114 min).",202 "handling": "Not used; the extracted quantity is the Table III Model 2 coefficient, which is internally consistent with the Sec. VI-C CI and p-value."},203 {"issue": "S1: Table III states n = 96 while Sec. VI-B reports N = 47 + 46 = 93 for time on task.",204 "handling": "Both reported verbatim in sample_as_printed; does not affect theta or SE."},205 {"issue": "S1: printed SE 0.14 vs SE implied by the printed 95% CI [-0.51, 0.03] = 0.1378 (rounding).",206 "handling": "Printed SE(b)=0.14 used per convention; CI-implied SE recorded in transformation as cross-check."},207 {"issue": "S1: Table III rendering collapses four model columns into one line with the variable name at the end and garbles the 'Adj. R^2' / 'p value' row labels.",208 "handling": "Columns read left to right as Models 1-4 per the editorial note; Model 2 = -0.24 confirmed by text (Sec. VI-C)."},209 {"issue": "S2 (Becker): the confidence interval of the headline 19% estimate is only displayed in Figure 1, not printed in the extract.",210 "handling": "Harmonised (theta = ln 1.19) but not poolable; no remembered value substituted."},211 {"issue": "S2: a raw ratio estimate of 34% is also printed (footnote 12), differing from the 19% regression estimate.",212 "handling": "The regression-based 19% headline estimate requested in studies.json is used."},213 {"issue": "S3 (Bono & Xu): printed Improvement -29.79% vs ratio of printed means 44.29/63.10 - 1 = -29.81%.",214 "handling": "Printed percentage used (difference 0.0003 on theta scale, rounding)."},215 {"issue": "S3: only significance stars are printed, with a nonstandard definition (* 0.05<p<=0.1, ** 0.01<p<=0.05, *** p<=0.01).",216 "handling": "SE not recoverable; study marked not poolable."},217 {"issue": "S4 (Bono 2025): printed % Time Savings 11.1 vs 1 - 50.5/56.9 = 11.25% from printed means.",218 "handling": "Printed percentage and p-value used; sensitivity with the ratio of means reported in results (sensitivity_s4_ratio_of_means)."},219 {"issue": "S4: Table 3 Malicious/Non-Malicious 'time per email (s)' values (1.8-2.8) are implausibly small vs the Full Corpus (~50 s), and the Malicious % (-52.9) does not match 2.8/1.9; units ambiguous in those columns.",220 "handling": "Only the Full Corpus column is used; anomaly documented."},221 {"issue": "S4: arm sizes (52/56/59, summing to 167) appear to be before the filtering of 13 responses; analysed N per arm not printed.",222 "handling": "Reported verbatim in sample_as_printed; not needed for theta or the p-value-based SE."},223 {"issue": "S4: SE is approximate, recovered from a two-sided p-value (0.471) assuming normality.",224 "handling": "Marked as approximate; S4 enters pooling per the stated convention."},225 {"issue": "S5 (Kreitmeir & Raschky): table notes print '** p<0.5' (presumably p<0.05); row label rendered as '\\times TreatPost'; date line 'August 24, 2026' is a rendering artefact.",226 "handling": "No effect: S5 is not harmonisable."},227]228229# sensitivity (not part of required keys): S4 theta from ratio of printed means, same z230theta4_alt = math.log(aw4 / ct4)231se4_alt = abs(theta4_alt) / z4232w_alt = [1 / se1 ** 2, 1 / se4_alt ** 2]233fe_alt = (w_alt[0] * theta1 + w_alt[1] * theta4_alt) / sum(w_alt)234# sensitivity: S1 SE from CI235w_ci = [1 / se1_from_ci ** 2, 1 / se4 ** 2]236fe_ci = (w_ci[0] * theta1 + w_ci[1] * theta4) / sum(w_ci)237238results = {239 "task_id": "T3-synthesis",240 "target_definition": "theta = ln(T_AI / T_noAI): natural log of the ratio of mean task-completion time with access to the generative-AI assistant to mean completion time without it, on the same tasks, from a randomised comparison; negative = faster with AI.",241 "items": items,242 "k_extracted": sum(1 for v in items.values() if v["estimate_as_printed"]),243 "k_harmonised": len(harm),244 "k_pooled": len(pool),245 "pooled_studies": [p[0] for p in pool],246 "fe_estimate": fe, "fe_se": fe_se, "fe_ci_low": fe - Z * fe_se, "fe_ci_high": fe + Z * fe_se,247 "re_estimate": re_est, "re_se": re_se, "re_ci_low": re_lo, "re_ci_high": re_hi,248 "q_stat": Q, "q_df": df, "q_pvalue": float(1 - __import__("scipy.stats", fromlist=["chi2"]).chi2.cdf(Q, df)),249 "i2": i2, "tau2": tau2,250 "heterogeneity_interpretation": het,251 "pooled_significant_5pct": bool(re_lo > 0 or re_hi < 0),252 "pooled_pct_change_re": 100 * (math.exp(re_est) - 1),253 "baseline_unweighted_mean_poolable": sum(y) / len(y),254 "baseline_unweighted_mean_harmonised": sum(harm.values()) / len(harm),255 "sensitivity_s4_ratio_of_means": {"theta_S4": theta4_alt, "se_S4": se4_alt, "fe_estimate": fe_alt},256 "sensitivity_s1_se_from_ci": {"se_S1": se1_from_ci, "fe_estimate": fe_ci},257 "feasibility": "complete",258 "feasibility_reason": None,259 "data_issues": data_issues,260 "n_input_rows": n_input_rows,261 "n_analysed": len(items),262 "random_seed": None,263}264265with open(os.path.join(OUT, "results.json"), "w") as fh:266 json.dump(results, fh, indent=1)267268cols = ["study", "estimate_as_printed", "uncertainty_as_printed", "sample_as_printed", "definition", "locator",269 "harmonised_estimate", "harmonised_se", "harmonisable", "poolable", "transformation", "reason_if_not"]270with open(os.path.join(OUT, "table1.csv"), "w", newline="") as fh:271 wr_ = csv.writer(fh)272 wr_.writerow(cols)273 for k, v in items.items():274 wr_.writerow([k] + [("" if v[c] is None else v[c]) for c in cols[1:]])275276# ---------------- Forest plot ----------------277short = {"S1": "S1 Paradis et al. 2024 (Google)", "S2": "S2 Becker et al. 2025 (METR)",278 "S3": "S3 Bono & Xu 2024 (Security Copilot)", "S4": "S4 Bono 2025 (Phishing agent)"}279rows = [k for k in items if items[k]["harmonisable"]]280fig, ax = plt.subplots(figsize=(9.5, 4.2))281ypos = list(range(len(rows) + 1, 1, -1))282for yp, k in zip(ypos, rows):283 t, s = items[k]["harmonised_estimate"], items[k]["harmonised_se"]284 if s is not None:285 ax.errorbar(t, yp, xerr=Z * s, fmt="s", color="black", capsize=3)286 lab = f"{t:+.3f} [{t - Z*s:+.3f}, {t + Z*s:+.3f}]" + (" (approx. SE)" if k == "S4" else "")287 else:288 ax.plot(t, yp, "o", mfc="white", color="grey")289 lab = f"{t:+.3f} (no SE; not pooled)"290 ax.text(1.02, yp, lab, transform=ax.get_yaxis_transform(), va="center", fontsize=8)291ax.fill([re_lo, re_est, re_hi, re_est], [1, 1.25, 1, 0.75], color="tab:blue")292ax.text(1.02, 1, f"{re_est:+.3f} [{re_lo:+.3f}, {re_hi:+.3f}]", transform=ax.get_yaxis_transform(), va="center", fontsize=8)293ax.axvline(0, color="grey", ls="--", lw=0.8)294ax.set_yticks(ypos + [1])295ax.set_yticklabels([short[k] for k in rows] + [f"RE pooled (DL), k={len(pool)}"])296ax.set_xlabel("theta = ln(T_AI / T_noAI) (negative = faster with AI)")297ax.set_title("Forest plot: harmonised effects of AI access on completion time")298ax.set_ylim(0.3, len(rows) + 1.7)299fig.subplots_adjust(left=0.30, right=0.74)300fig.savefig(os.path.join(OUT, "figure1.png"), dpi=150)301302print("=== Results ===")303for k, v in items.items():304 print(k, "theta =", v["harmonised_estimate"], "SE =", v["harmonised_se"], "poolable =", v["poolable"])305print(f"FE {fe:.4f} (SE {fe_se:.4f}); RE {re_est:.4f} [{re_lo:.4f}, {re_hi:.4f}]; Q {Q:.4f}; I2 {i2:.2f}; tau2 {tau2:.5f}; "306 f"pct change RE {results['pooled_pct_change_re']:.2f}%")307