"""T3-synthesis: extraction, harmonisation to theta = ln(T_AI/T_noAI), FE + DL pooling. Run from workspace root: ~/wp29/venv/bin/python out/analysis.py Reads only data/; writes out/results.json, out/table1.csv, out/figure1.png. Deterministic.""" import json, math, re, os import numpy as np, pandas as pd from scipy.stats import norm import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt D = "data"; SRC = os.path.join(D, "sources"); OUT = "out" os.makedirs(OUT, exist_ok=True) studies = json.load(open(os.path.join(D, "studies.json"))) def lines(src): return open(os.path.join(SRC, src + ".txt"), encoding="utf-8").read().split("\n") def find(src, pattern, start=0): """Return (1-based line number, text) of first line matching regex at/after `start`; print it.""" L = lines(src) for i in range(start, len(L)): if re.search(pattern, L[i]): print(f"[{src}:{i+1}] {L[i].strip()[:300]}") return i + 1, L[i] raise ValueError(f"pattern {pattern!r} not found in {src}") def nums(s): return re.findall(r"-?\d+\.?\d*", s) items = {} print("=== Source lines relied upon ===") # ---------------- S1 Paradis et al. 2024 ---------------- s = "arxiv-2410.12944v3" n_tab, l_tab = find(s, r"TABLE III: Summaries") n_b, l_b = find(s, r"ExpCon \|") n_se, l_se = find(s, r"^\(0\.\d+\) \(0\.\d+\) \(0\.\d+\) \(0\.\d+\) \|", n_b) n_ci, l_ci = find(s, r"best-fit Model 2") find(s, r"Mean LogToT") # printed for data_issues b_tokens = l_b.split()[:4] # '-0.25.' '-0.24.' '-0.30*' '-0.14' se_tokens = re.findall(r"\(([\d.]+)\)", l_se) b_print = b_tokens[1] # Model 2 column b2 = float(re.sub(r"[^\d.\-]", "", b_print).rstrip(".")) se2_print = se_tokens[1]; se2 = float(se2_print) ci = re.search(r"95\\%CI = \[(-?[\d.]+),(-?[\d.]+)\]", l_ci); L1, U1 = float(ci.group(1)), float(ci.group(2)) p1 = re.search(r"p=([\d.]+)", l_ci).group(1) se_from_ci = (U1 - L1) / (2 * 1.96) items["S1"] = dict( estimate_as_printed=f"ExpCon (Model 2) = {b_print.rstrip('.')} (marker '.', p<.1) on LogToT", uncertainty_as_printed=f"SE ({se2_print}) [Table III, Model 2]; 95%CI = [{ci.group(1)},{ci.group(2)}], p={p1} [Section VI-C]", sample_as_printed="n = 96 (Table III caption); N=47 AI, N=46 no-AI in Section VI-B", definition="OLS coefficient on experimental condition (AI vs no-AI) with natural-log time on task as dependent variable, controlling for developer-level factors (Model 2)", locator=f"{s}, Table III (line {n_b}: ExpCon row, 2nd value = Model 2; line {n_se}: SE row), Section VI-C (line {n_ci})", harmonisable=True, harmonised_estimate=b2, harmonised_se=se2, poolable=True, transformation=f"theta = b = {b2}; SE(theta) = SE(b) = {se2} (printed). Cross-check from CI: ({U1} - ({L1}))/(2*1.96) = {se_from_ci:.4f}", reason_if_not=None) # ---------------- S2 Becker et al. 2025 ---------------- s = "arxiv-2507.09089v2" n2, l2 = find(s, r"implement issues in 19% more time") n2b, _ = find(s, r"allowing AI actually increases completion time by 19%") n2c, _ = find(s, r"Developers complete 136 issues") n2d, _ = find(s, r"By default, we report 95% confidence intervals") n2e, _ = find(s, r"Omitted:") c2 = float(re.search(r"(\d+)% more time", l2).group(1)) / 100 theta2 = math.log(1 + c2) items["S2"] = dict( estimate_as_printed="19% more time (issue completion time, AI allowed vs disallowed; regression-based)", uncertainty_as_printed="none printed in extract (95% CI using HC3 SEs is shown only in Figure 1, not reproduced)", sample_as_printed="16 developers; 246 tasks (136 issues AI-allowed, 110 AI-disallowed)", definition="S = E[time AI allowed]/E[time AI disallowed] - 1, estimated as exp(beta)-1 from log-linear OLS of completion time on AI indicator and log forecast time", locator=f"{s}, Section 3 Results (line {n2}); Abstract (line {n2b}); estimator Sections 2.3, D.1-D.3; CI method D.2 (line {n2d})", harmonisable=True, harmonised_estimate=theta2, harmonised_se=None, poolable=False, transformation=f"theta = ln(1 + c) = ln(1 + {c2}) = {theta2:.4f}", reason_if_not="Not poolable: no SE, CI or p-value for the 19% estimate is printed in the extract; the confidence interval appears only in Figure 1 (not reproduced), so SE(theta) cannot be obtained.") # ---------------- S3 Bono & Xu 2024 ---------------- s = "arxiv-2411.01067v2" n3h, _ = find(s, r"\| Overall \| Treatment \| Control \| Improvement") n3a, _ = find(s, r"^Aggregate Results") n3, l3 = find(s, r"^All Content \|", n3a) n3c, _ = find(s, r"Table 3: Comparison of task duration") n3e, _ = find(s, r"Editorial note") n3n, _ = find(s, r"We had 181 subjects") cells = [c.strip() for c in l3.split("|")] overall3, treat3, ctrl3, impr3 = cells[1], cells[2], cells[3], cells[4] c3 = float(re.sub(r"[^\d.\-]", "", impr3)) / 100 theta3 = math.log(1 + c3) theta3_ratio = math.log(float(treat3) / float(ctrl3)) items["S3"] = dict( estimate_as_printed=f"All Content (aggregate): Overall {overall3}, Treatment {treat3}, Control {ctrl3} minutes; Improvement {impr3}", uncertainty_as_printed="significance stars only (*** = 0 change in time c = -0.111 theta4 = math.log(1 + c4) z4 = norm.ppf(1 - float(p4) / 2) se4 = abs(theta4) / z4 items["S4"] = dict( estimate_as_printed=f"Full Corpus: Aware Time per Email (s) {aware4}; Control Time per Email (s) {ctrl4}; % Time Savings {sav4}", uncertainty_as_printed=f"p-value {p4}", sample_as_printed="Aware N=56, Control N=52 (Blind N=59 not used); 25-email queue per subject", definition="Percentage time saving in mean processing time per email (full queue), Aware arm (agent verdicts + rationale + priority ordering) vs Control (no AI)", locator=f"{s}, Section 3.3, Table 3 (lines {n4a}-{n4p}; column 'Full Corpus')", harmonisable=True, harmonised_estimate=theta4, harmonised_se=se4, poolable=True, transformation=(f"c = -{sav4}/100 = {c4}; theta = ln(1 + c) = ln({1 + c4:.3f}) = {theta4:.4f} " f"(cross-check ln({aware4}/{ctrl4}) = {math.log(float(aware4)/float(ctrl4)):.4f}); " f"APPROXIMATE SE = |theta|/z_(1-p/2) = {abs(theta4):.4f}/{z4:.4f} = {se4:.4f} from printed p = {p4}"), reason_if_not=None) # ---------------- S5 Kreitmeir & Raschky 2023 ---------------- s = "arxiv-2304.09339v1" n5a, _ = find(s, r"^\| A: Pre 27/28.03") n5, l5 = find(s, r"TreatPost \|", n5a) n5s, l5s = find(s, r"^\| \(", n5) n5n, l5n = find(s, r"^N \|", n5) n5f, _ = find(s, r"\*\* p<0\.5") b5 = [c.strip() for c in l5.split("|")][3] se5 = [c.strip() for c in l5s.split("|")][3] N5 = [c.strip() for c in l5n.split("|")][3] items["S5"] = dict( estimate_as_printed=f"Treat x Post (Output, column 3, panel A) = {b5}", uncertainty_as_printed=f"SE {se5} (clustered by user); no significance star", sample_as_printed=f"N = {N5} (user-days, users with affiliation, panel A)", definition="Difference-in-differences coefficient on an aggregate GitHub output count (Push, PRs, comments, Create, Issues) for Italian vs Austrian/French users, around the ChatGPT ban (withdrawal of access)", locator=f"{s}, Section 4, Table 1, panel A, row 'Treat x Post' (line {n5}), SE line {n5s}, N line {n5n}, column (3) Output", harmonisable=False, harmonised_estimate=None, harmonised_se=None, poolable=False, transformation="none (not harmonisable)", reason_if_not="Not harmonisable: the outcome is a count of GitHub output events per user-day (activity volume), not task-completion time on the same tasks; the contrast is country-level withdrawal of access (DiD), not a randomised AI-vs-no-AI comparison of time on identical tasks.") # ---------------- Pooling ---------------- order = ["S1", "S2", "S3", "S4", "S5"] pool = [k for k in order if items[k]["poolable"]] y = np.array([items[k]["harmonised_estimate"] for k in pool]) se = np.array([items[k]["harmonised_se"] for k in pool]) def meta(y, se): w = 1 / se**2 fe = np.sum(w * y) / np.sum(w); fe_se = math.sqrt(1 / np.sum(w)) Q = float(np.sum(w * (y - fe) ** 2)); df = len(y) - 1 i2 = max(0.0, (Q - df) / Q) * 100 if Q > 0 else 0.0 C = np.sum(w) - np.sum(w**2) / np.sum(w) tau2 = max(0.0, (Q - df) / C) wr = 1 / (se**2 + tau2) re_ = np.sum(wr * y) / np.sum(wr); re_se = math.sqrt(1 / np.sum(wr)) return dict(fe=float(fe), fe_se=fe_se, Q=Q, i2=float(i2), tau2=float(tau2), re=float(re_), re_se=re_se) m = meta(y, se) # sensitivity: S1 SE from the printed CI instead of printed SE se_alt = se.copy(); se_alt[pool.index("S1")] = se_from_ci m_alt = meta(y, se_alt) i2 = m["i2"] het = "low" if i2 < 25 else ("moderate" if i2 < 50 else "substantial") re_lo, re_hi = m["re"] - 1.96 * m["re_se"], m["re"] + 1.96 * m["re_se"] data_issues = [ {"issue": "S1 (arxiv-2410.12944v3, Section VI-B): the printed mean LogToT is 4.6 for the AI group and 4.3 for the no-AI group, which contradicts the printed mean times (96 min AI vs 114 min no-AI; ln 96 = 4.56, ln 114 = 4.74) and the statement that the AI group was faster.", "handling": "Not used. The harmonised value is the Model 2 regression coefficient specified in studies.json, whose sign agrees with the mean times and with the text."}, {"issue": "S1: Table III caption says n = 96, whereas Section VI-B reports N=47 (AI) + N=46 (no-AI) = 93 for the time comparison.", "handling": "Both reported as printed in sample_as_printed; the SE is taken as printed, so the discrepancy does not affect theta or SE."}, {"issue": "S1: Table III renders the four model columns collapsed into one line, with significance markers attached to the numbers (e.g. '-0.24.'). The trailing '.' is the p<.1 marker, not a decimal point. The printed SE (0.14) and the SE implied by the printed 95% CI [-0.51, 0.03] (0.1378) differ slightly because of rounding.", "handling": "Model 2 = second value on the ExpCon line; the marker was stripped. The printed SE 0.14 was used, and a sensitivity analysis uses the CI-derived SE."}, {"issue": "S1: the extract has stray 'p<' / 'p=' lines and footnote digits ('^2^2', '2').", "handling": "Ignored as rendering artefacts (data/README.md)."}, {"issue": "S2: the uncertainty of the headline 19% estimate is shown only in Figure 1 (not reproduced). The raw ratio of means (34%) differs from the regression-based 19%.", "handling": "Used the regression-based 19%, as studies.json specifies. Harmonised theta = ln(1.19). Not poolable because there is no SE."}, {"issue": "S3: the Table 3 header row has four labels ('Overall | Treatment | Control | Improvement') with an empty leading cell. The 'Improvement' column is actually a signed % change in time (negative = faster); for MC rows a positive value is a time increase (per the text). The printed -29.79% matches 44.29/63.10 - 1 = -0.2981 only up to rounding.", "handling": "Read the 'Improvement' value as the signed % change c = -0.2979. Computed theta from the printed percentage and reported ln(44.29/63.10) as a cross-check."}, {"issue": "S3: the significance-star convention is non-standard (* 0.05 0 or re_hi < 0), pooled_pct_change_re=100 * (math.exp(m["re"]) - 1), feasibility="complete", feasibility_reason=None, data_issues=data_issues, n_input_rows=n_input_rows, n_analysed=len(order), random_seed=None, sensitivity_s1_se_from_ci=dict(s1_se=se_from_ci, fe_estimate=m_alt["fe"], fe_se=m_alt["fe_se"], re_estimate=m_alt["re"], re_se=m_alt["re_se"], q_stat=m_alt["Q"], i2=m_alt["i2"], tau2=m_alt["tau2"]), ) with open(os.path.join(OUT, "results.json"), "w") as fh: json.dump(res, fh, indent=2) cols = ["estimate_as_printed", "uncertainty_as_printed", "sample_as_printed", "definition", "locator", "harmonised_estimate", "harmonised_se", "harmonisable", "poolable", "transformation", "reason_if_not"] pd.DataFrame([dict(study=k, **{c: items[k][c] for c in cols}) for k in order]).to_csv(os.path.join(OUT, "table1.csv"), index=False) # ---------------- Forest plot ---------------- harm = [k for k in order if items[k]["harmonisable"]] fig, ax = plt.subplots(figsize=(8, 4.2)) labels = [] for i, k in enumerate(harm): yv = len(harm) - i t = items[k]["harmonised_estimate"]; s_ = items[k]["harmonised_se"] if s_ is not None: ax.errorbar(t, yv, xerr=1.96 * s_, fmt="s", color="k", capsize=3) lab = f"{k}: {t:.3f} [{t-1.96*s_:.3f}, {t+1.96*s_:.3f}]" + (" (approx. SE)" if k == "S4" else "") else: ax.plot(t, yv, "o", mfc="white", mec="gray") lab = f"{k}: {t:.3f} (no SE; not pooled)" labels.append(lab) ax.errorbar(m["re"], 0, xerr=1.96 * m["re_se"], fmt="D", color="tab:blue", capsize=3) labels.append(f"RE pooled (k={len(pool)}): {m['re']:.3f} [{re_lo:.3f}, {re_hi:.3f}]") ax.set_yticks(list(range(len(harm), -1, -1))); ax.set_yticklabels(labels, fontsize=8) ax.axvline(0, color="gray", ls="--", lw=0.8) ax.set_xlabel("theta = ln(T_AI / T_noAI) (negative = faster with AI)") ax.set_title("Harmonised effects of generative-AI access on task time (95% CI)", fontsize=10) fig.tight_layout(); fig.savefig(os.path.join(OUT, "figure1.png"), dpi=150) print("\n=== Summary ===") for k in order: print(k, items[k]["harmonised_estimate"], items[k]["harmonised_se"], items[k]["poolable"]) print(json.dumps({k: res[k] for k in ["k_pooled", "fe_estimate", "fe_se", "re_estimate", "re_se", "re_ci_low", "re_ci_high", "q_stat", "i2", "tau2", "pooled_pct_change_re", "pooled_significant_5pct"]}, indent=1)) print("sensitivity:", res["sensitivity_s1_se_from_ci"])