1"""T3-synthesis: extraction, harmonisation to theta = ln(T_AI/T_noAI), FE + DL pooling.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Reads only data/; writes out/results.json, out/table1.csv, out/figure1.png. Deterministic."""4import json, math, re, os5import numpy as np, pandas as pd6from scipy.stats import norm7import matplotlib8matplotlib.use("Agg")9import matplotlib.pyplot as plt1011D = "data"; SRC = os.path.join(D, "sources"); OUT = "out"12os.makedirs(OUT, exist_ok=True)13studies = json.load(open(os.path.join(D, "studies.json")))1415def lines(src):16 return open(os.path.join(SRC, src + ".txt"), encoding="utf-8").read().split("\n")1718def find(src, pattern, start=0):19 """Return (1-based line number, text) of first line matching regex at/after `start`; print it."""20 L = lines(src)21 for i in range(start, len(L)):22 if re.search(pattern, L[i]):23 print(f"[{src}:{i+1}] {L[i].strip()[:300]}")24 return i + 1, L[i]25 raise ValueError(f"pattern {pattern!r} not found in {src}")2627def nums(s):28 return re.findall(r"-?\d+\.?\d*", s)2930items = {}31print("=== Source lines relied upon ===")3233# ---------------- S1 Paradis et al. 2024 ----------------34s = "arxiv-2410.12944v3"35n_tab, l_tab = find(s, r"TABLE III: Summaries")36n_b, l_b = find(s, r"ExpCon \|")37n_se, l_se = find(s, r"^\(0\.\d+\) \(0\.\d+\) \(0\.\d+\) \(0\.\d+\) \|", n_b)38n_ci, l_ci = find(s, r"best-fit Model 2")39find(s, r"Mean LogToT") # printed for data_issues40b_tokens = l_b.split()[:4] # '-0.25.' '-0.24.' '-0.30*' '-0.14'41se_tokens = re.findall(r"\(([\d.]+)\)", l_se)42b_print = b_tokens[1] # Model 2 column43b2 = float(re.sub(r"[^\d.\-]", "", b_print).rstrip("."))44se2_print = se_tokens[1]; se2 = float(se2_print)45ci = re.search(r"95\\%CI = \[(-?[\d.]+),(-?[\d.]+)\]", l_ci); L1, U1 = float(ci.group(1)), float(ci.group(2))46p1 = re.search(r"p=([\d.]+)", l_ci).group(1)47se_from_ci = (U1 - L1) / (2 * 1.96)48items["S1"] = dict(49 estimate_as_printed=f"ExpCon (Model 2) = {b_print.rstrip('.')} (marker '.', p<.1) on LogToT",50 uncertainty_as_printed=f"SE ({se2_print}) [Table III, Model 2]; 95%CI = [{ci.group(1)},{ci.group(2)}], p={p1} [Section VI-C]",51 sample_as_printed="n = 96 (Table III caption); N=47 AI, N=46 no-AI in Section VI-B",52 definition="OLS coefficient on experimental condition (AI vs no-AI) with natural-log time on task as dependent variable, controlling for developer-level factors (Model 2)",53 locator=f"{s}, Table III (line {n_b}: ExpCon row, 2nd value = Model 2; line {n_se}: SE row), Section VI-C (line {n_ci})",54 harmonisable=True, harmonised_estimate=b2, harmonised_se=se2, poolable=True,55 transformation=f"theta = b = {b2}; SE(theta) = SE(b) = {se2} (printed). Cross-check from CI: ({U1} - ({L1}))/(2*1.96) = {se_from_ci:.4f}",56 reason_if_not=None)5758# ---------------- S2 Becker et al. 2025 ----------------59s = "arxiv-2507.09089v2"60n2, l2 = find(s, r"implement issues in 19% more time")61n2b, _ = find(s, r"allowing AI actually increases completion time by 19%")62n2c, _ = find(s, r"Developers complete 136 issues")63n2d, _ = find(s, r"By default, we report 95% confidence intervals")64n2e, _ = find(s, r"Omitted:")65c2 = float(re.search(r"(\d+)% more time", l2).group(1)) / 10066theta2 = math.log(1 + c2)67items["S2"] = dict(68 estimate_as_printed="19% more time (issue completion time, AI allowed vs disallowed; regression-based)",69 uncertainty_as_printed="none printed in extract (95% CI using HC3 SEs is shown only in Figure 1, not reproduced)",70 sample_as_printed="16 developers; 246 tasks (136 issues AI-allowed, 110 AI-disallowed)",71 definition="S = E[time AI allowed]/E[time AI disallowed] - 1, estimated as exp(beta)-1 from log-linear OLS of completion time on AI indicator and log forecast time",72 locator=f"{s}, Section 3 Results (line {n2}); Abstract (line {n2b}); estimator Sections 2.3, D.1-D.3; CI method D.2 (line {n2d})",73 harmonisable=True, harmonised_estimate=theta2, harmonised_se=None, poolable=False,74 transformation=f"theta = ln(1 + c) = ln(1 + {c2}) = {theta2:.4f}",75 reason_if_not="Not poolable: no SE, CI or p-value for the 19% estimate is printed in the extract; the confidence interval appears only in Figure 1 (not reproduced), so SE(theta) cannot be obtained.")7677# ---------------- S3 Bono & Xu 2024 ----------------78s = "arxiv-2411.01067v2"79n3h, _ = find(s, r"\| Overall \| Treatment \| Control \| Improvement")80n3a, _ = find(s, r"^Aggregate Results")81n3, l3 = find(s, r"^All Content \|", n3a)82n3c, _ = find(s, r"Table 3: Comparison of task duration")83n3e, _ = find(s, r"Editorial note")84n3n, _ = find(s, r"We had 181 subjects")85cells = [c.strip() for c in l3.split("|")]86overall3, treat3, ctrl3, impr3 = cells[1], cells[2], cells[3], cells[4]87c3 = float(re.sub(r"[^\d.\-]", "", impr3)) / 10088theta3 = math.log(1 + c3)89theta3_ratio = math.log(float(treat3) / float(ctrl3))90items["S3"] = dict(91 estimate_as_printed=f"All Content (aggregate): Overall {overall3}, Treatment {treat3}, Control {ctrl3} minutes; Improvement {impr3}",92 uncertainty_as_printed="significance stars only (*** = 0<p<=0.01, per editorial note); no SE or CI printed",93 sample_as_printed="181 subjects completed the task (Section 3); treatment/control split not printed",94 definition="Percentage difference in mean task duration (minutes), Copilot vs control, holding accuracy constant (control duration predicted at treatment accuracy; Appendix B); bootstrap significance",95 locator=f"{s}, Table 3 (line {n3}: 'Aggregate Results / All Content' row; columns Treatment, Control, Improvement)",96 harmonisable=True, harmonised_estimate=theta3, harmonised_se=None, poolable=False,97 transformation=f"theta = ln(1 + c) = ln(1 + ({c3})) = {theta3:.4f} (cross-check ln({treat3}/{ctrl3}) = {theta3_ratio:.4f})",98 reason_if_not="Not poolable: uncertainty is reported only as significance stars (***, p<=0.01); a p-value threshold does not identify SE(theta).")99100# ---------------- S4 Bono 2025 ----------------101s = "arxiv-2511.13860v1"102n4h, l4h = find(s, r"^Metric \| Malicious")103n4a, l4a = find(s, r"^Aware Time per Email")104n4c, l4c = find(s, r"^Control Time per Email")105n4s, l4s = find(s, r"^% Time Savings")106n4p, l4p = find(s, r"^p-value \|")107n4n, _ = find(s, r"Control \( N\{=\}52")108find(s, r"Aware \( N\{=\}56")109last = lambda l: l.split("|")[-1].strip()110aware4, ctrl4, sav4, p4 = last(l4a), last(l4c), last(l4s), last(l4p)111c4 = -float(sav4) / 100 # savings of 11.1% -> change in time c = -0.111112theta4 = math.log(1 + c4)113z4 = norm.ppf(1 - float(p4) / 2)114se4 = abs(theta4) / z4115items["S4"] = dict(116 estimate_as_printed=f"Full Corpus: Aware Time per Email (s) {aware4}; Control Time per Email (s) {ctrl4}; % Time Savings {sav4}",117 uncertainty_as_printed=f"p-value {p4}",118 sample_as_printed="Aware N=56, Control N=52 (Blind N=59 not used); 25-email queue per subject",119 definition="Percentage time saving in mean processing time per email (full queue), Aware arm (agent verdicts + rationale + priority ordering) vs Control (no AI)",120 locator=f"{s}, Section 3.3, Table 3 (lines {n4a}-{n4p}; column 'Full Corpus')",121 harmonisable=True, harmonised_estimate=theta4, harmonised_se=se4, poolable=True,122 transformation=(f"c = -{sav4}/100 = {c4}; theta = ln(1 + c) = ln({1 + c4:.3f}) = {theta4:.4f} "123 f"(cross-check ln({aware4}/{ctrl4}) = {math.log(float(aware4)/float(ctrl4)):.4f}); "124 f"APPROXIMATE SE = |theta|/z_(1-p/2) = {abs(theta4):.4f}/{z4:.4f} = {se4:.4f} from printed p = {p4}"),125 reason_if_not=None)126127# ---------------- S5 Kreitmeir & Raschky 2023 ----------------128s = "arxiv-2304.09339v1"129n5a, _ = find(s, r"^\| A: Pre 27/28.03")130n5, l5 = find(s, r"TreatPost \|", n5a)131n5s, l5s = find(s, r"^\| \(", n5)132n5n, l5n = find(s, r"^N \|", n5)133n5f, _ = find(s, r"\*\* p<0\.5")134b5 = [c.strip() for c in l5.split("|")][3]135se5 = [c.strip() for c in l5s.split("|")][3]136N5 = [c.strip() for c in l5n.split("|")][3]137items["S5"] = dict(138 estimate_as_printed=f"Treat x Post (Output, column 3, panel A) = {b5}",139 uncertainty_as_printed=f"SE {se5} (clustered by user); no significance star",140 sample_as_printed=f"N = {N5} (user-days, users with affiliation, panel A)",141 definition="Difference-in-differences coefficient on an aggregate GitHub output count (Push, PRs, comments, Create, Issues) for Italian vs Austrian/French users, around the ChatGPT ban (withdrawal of access)",142 locator=f"{s}, Section 4, Table 1, panel A, row 'Treat x Post' (line {n5}), SE line {n5s}, N line {n5n}, column (3) Output",143 harmonisable=False, harmonised_estimate=None, harmonised_se=None, poolable=False,144 transformation="none (not harmonisable)",145 reason_if_not="Not harmonisable: the outcome is a count of GitHub output events per user-day (activity volume), not task-completion time on the same tasks; the contrast is country-level withdrawal of access (DiD), not a randomised AI-vs-no-AI comparison of time on identical tasks.")146147# ---------------- Pooling ----------------148order = ["S1", "S2", "S3", "S4", "S5"]149pool = [k for k in order if items[k]["poolable"]]150y = np.array([items[k]["harmonised_estimate"] for k in pool])151se = np.array([items[k]["harmonised_se"] for k in pool])152153def meta(y, se):154 w = 1 / se**2155 fe = np.sum(w * y) / np.sum(w); fe_se = math.sqrt(1 / np.sum(w))156 Q = float(np.sum(w * (y - fe) ** 2)); df = len(y) - 1157 i2 = max(0.0, (Q - df) / Q) * 100 if Q > 0 else 0.0158 C = np.sum(w) - np.sum(w**2) / np.sum(w)159 tau2 = max(0.0, (Q - df) / C)160 wr = 1 / (se**2 + tau2)161 re_ = np.sum(wr * y) / np.sum(wr); re_se = math.sqrt(1 / np.sum(wr))162 return dict(fe=float(fe), fe_se=fe_se, Q=Q, i2=float(i2), tau2=float(tau2), re=float(re_), re_se=re_se)163164m = meta(y, se)165# sensitivity: S1 SE from the printed CI instead of printed SE166se_alt = se.copy(); se_alt[pool.index("S1")] = se_from_ci167m_alt = meta(y, se_alt)168169i2 = m["i2"]170het = "low" if i2 < 25 else ("moderate" if i2 < 50 else "substantial")171re_lo, re_hi = m["re"] - 1.96 * m["re_se"], m["re"] + 1.96 * m["re_se"]172173data_issues = [174 {"issue": "S1 (arxiv-2410.12944v3, Section VI-B): the printed mean LogToT is 4.6 for the AI group and 4.3 for the no-AI group, which contradicts the printed mean times (96 min AI vs 114 min no-AI; ln 96 = 4.56, ln 114 = 4.74) and the statement that the AI group was faster.",175 "handling": "Not used. The harmonised value is the Model 2 regression coefficient specified in studies.json, whose sign agrees with the mean times and with the text."},176 {"issue": "S1: Table III caption says n = 96, whereas Section VI-B reports N=47 (AI) + N=46 (no-AI) = 93 for the time comparison.",177 "handling": "Both reported as printed in sample_as_printed; the SE is taken as printed, so the discrepancy does not affect theta or SE."},178 {"issue": "S1: Table III renders the four model columns collapsed into one line, with significance markers attached to the numbers (e.g. '-0.24.'). The trailing '.' is the p<.1 marker, not a decimal point. The printed SE (0.14) and the SE implied by the printed 95% CI [-0.51, 0.03] (0.1378) differ slightly because of rounding.",179 "handling": "Model 2 = second value on the ExpCon line; the marker was stripped. The printed SE 0.14 was used, and a sensitivity analysis uses the CI-derived SE."},180 {"issue": "S1: the extract has stray 'p<' / 'p=' lines and footnote digits ('^2^2', '2').",181 "handling": "Ignored as rendering artefacts (data/README.md)."},182 {"issue": "S2: the uncertainty of the headline 19% estimate is shown only in Figure 1 (not reproduced). The raw ratio of means (34%) differs from the regression-based 19%.",183 "handling": "Used the regression-based 19%, as studies.json specifies. Harmonised theta = ln(1.19). Not poolable because there is no SE."},184 {"issue": "S3: the Table 3 header row has four labels ('Overall | Treatment | Control | Improvement') with an empty leading cell. The 'Improvement' column is actually a signed % change in time (negative = faster); for MC rows a positive value is a time increase (per the text). The printed -29.79% matches 44.29/63.10 - 1 = -0.2981 only up to rounding.",185 "handling": "Read the 'Improvement' value as the signed % change c = -0.2979. Computed theta from the printed percentage and reported ln(44.29/63.10) as a cross-check."},186 {"issue": "S3: the significance-star convention is non-standard (* 0.05<p<=0.1, ** 0.01<p<=0.05, *** p<=0.01).",187 "handling": "Recorded. Stars give only a p-value threshold, so S3 is not poolable."},188 {"issue": "S4: Table 3 labels all columns 'Time per Email (s)', but the Malicious (2.8 / 1.9) and Non-Malicious (1.8 / 2.4) values cannot be seconds if the Full Corpus value is 50.5 / 56.9 s. No mix of the two categories gives the full-corpus mean in either seconds or minutes, so the units and/or rows are internally inconsistent.",189 "handling": "Only the Full Corpus column is used, and theta depends only on the unit-free ratio and % saving. The printed 11.1% agrees with 1 - 50.5/56.9 = 11.25% within rounding of the printed means. Flagged."},190 {"issue": "S4: the 'Aware' arm differs from Control in queue ordering as well as AI verdicts/rationales, and its % time saving is printed positive for a decrease in time (sign opposite to theta).",191 "handling": "Aware vs Control is taken as the AI vs no-AI contrast, and the sign is converted (c = -0.111). The SE is approximate, back-calculated from p = 0.471."},192 {"issue": "S5: the Table 1 footnote prints '** p<0.5' (presumably p<0.05). The extract's date line 'August 24, 2026' is a rendering-time artefact.",193 "handling": "Recorded. This does not affect anything because S5 is not harmonisable."},194]195196n_input_rows = {"studies.json": len(studies)}197for f in ["README.md"] + [f"sources/{x}" for x in sorted(os.listdir(SRC))]:198 with open(os.path.join(D, f), "rb") as fh:199 n_input_rows[f] = fh.read().count(b"\n")200201res = dict(202 task_id="T3-synthesis",203 target_definition="theta = ln(T_AI / T_noAI): natural log of the ratio of mean task-completion time with generative-AI assistance to mean completion time without it, on the same tasks, from a randomised comparison (negative = faster with AI).",204 items={k: items[k] for k in order},205 k_extracted=sum(1 for k in order if items[k]["estimate_as_printed"]),206 k_harmonised=sum(1 for k in order if items[k]["harmonised_estimate"] is not None),207 k_pooled=len(pool),208 fe_estimate=m["fe"], fe_se=m["fe_se"], fe_ci_low=m["fe"] - 1.96 * m["fe_se"], fe_ci_high=m["fe"] + 1.96 * m["fe_se"],209 re_estimate=m["re"], re_se=m["re_se"], re_ci_low=re_lo, re_ci_high=re_hi,210 q_stat=m["Q"], i2=i2, tau2=m["tau2"],211 heterogeneity_interpretation=het,212 pooled_significant_5pct=bool(re_lo > 0 or re_hi < 0),213 pooled_pct_change_re=100 * (math.exp(m["re"]) - 1),214 feasibility="complete",215 feasibility_reason=None,216 data_issues=data_issues,217 n_input_rows=n_input_rows,218 n_analysed=len(order),219 random_seed=None,220 sensitivity_s1_se_from_ci=dict(s1_se=se_from_ci, fe_estimate=m_alt["fe"], fe_se=m_alt["fe_se"],221 re_estimate=m_alt["re"], re_se=m_alt["re_se"], q_stat=m_alt["Q"], i2=m_alt["i2"], tau2=m_alt["tau2"]),222)223with open(os.path.join(OUT, "results.json"), "w") as fh:224 json.dump(res, fh, indent=2)225226cols = ["estimate_as_printed", "uncertainty_as_printed", "sample_as_printed", "definition", "locator",227 "harmonised_estimate", "harmonised_se", "harmonisable", "poolable", "transformation", "reason_if_not"]228pd.DataFrame([dict(study=k, **{c: items[k][c] for c in cols}) for k in order]).to_csv(os.path.join(OUT, "table1.csv"), index=False)229230# ---------------- Forest plot ----------------231harm = [k for k in order if items[k]["harmonisable"]]232fig, ax = plt.subplots(figsize=(8, 4.2))233labels = []234for i, k in enumerate(harm):235 yv = len(harm) - i236 t = items[k]["harmonised_estimate"]; s_ = items[k]["harmonised_se"]237 if s_ is not None:238 ax.errorbar(t, yv, xerr=1.96 * s_, fmt="s", color="k", capsize=3)239 lab = f"{k}: {t:.3f} [{t-1.96*s_:.3f}, {t+1.96*s_:.3f}]" + (" (approx. SE)" if k == "S4" else "")240 else:241 ax.plot(t, yv, "o", mfc="white", mec="gray")242 lab = f"{k}: {t:.3f} (no SE; not pooled)"243 labels.append(lab)244ax.errorbar(m["re"], 0, xerr=1.96 * m["re_se"], fmt="D", color="tab:blue", capsize=3)245labels.append(f"RE pooled (k={len(pool)}): {m['re']:.3f} [{re_lo:.3f}, {re_hi:.3f}]")246ax.set_yticks(list(range(len(harm), -1, -1))); ax.set_yticklabels(labels, fontsize=8)247ax.axvline(0, color="gray", ls="--", lw=0.8)248ax.set_xlabel("theta = ln(T_AI / T_noAI) (negative = faster with AI)")249ax.set_title("Harmonised effects of generative-AI access on task time (95% CI)", fontsize=10)250fig.tight_layout(); fig.savefig(os.path.join(OUT, "figure1.png"), dpi=150)251252print("\n=== Summary ===")253for k in order:254 print(k, items[k]["harmonised_estimate"], items[k]["harmonised_se"], items[k]["poolable"])255print(json.dumps({k: res[k] for k in ["k_pooled", "fe_estimate", "fe_se", "re_estimate", "re_se", "re_ci_low", "re_ci_high",256 "q_stat", "i2", "tau2", "pooled_pct_change_re", "pooled_significant_5pct"]}, indent=1))257print("sensitivity:", res["sensitivity_s1_se_from_ci"])258