1#!/usr/bin/env python32"""WP44 — preregistered known-emergence set (protocol P1, H8) as keyword-defined monthly series (OpenAlex topics are too3coarse for these concepts: e.g. no topic for LLM agents, VLMs, speculative decoding, MoE). For each concept, monthly counts of4eligible month-precision works whose title or abstract matches the regex; detection = first month at which the trailing53-month z-score against the trailing 24 months exceeds 3 AND 12-month growth > 100 % with >= 30 works; lead time =6peak-growth month (max 12-month log growth) minus detection month. Output: results/emerging/known_set_lead_times.csv."""7import argparse, os, re, duckdb, numpy as np, pandas as pd89KNOWN = {10 "large language models": r"\b(large language models?|LLMs?)\b",11 "diffusion models": r"\b(diffusion models?|denoising diffusion|latent diffusion|score-based generative)\b",12 "neural radiance fields": r"\b(neural radiance fields?|NeRFs?)\b",13 "llm agents": r"\b(LLM[- ]based agents?|language (model|agent)s? agents?|LLM agents?|agentic (AI|systems?|workflows?)|tool-using (agents?|LLMs?))\b",14 "retrieval-augmented generation": r"\b(retrieval[- ]augmented generation|\bRAG\b)",15 "state-space sequence models": r"\b(state[- ]space models?|Mamba|S4\b|selective state space|structured state space)\b",16 "vision-language models": r"\b(vision[- ]language models?|VLMs?|multimodal large language models?|MLLMs?)\b",17 "mechanistic interpretability": r"\b(mechanistic interpretability|sparse autoencoders? (for|in) (interpretab|LLM|language)|circuits? in (language|transformer))",18 "speculative decoding": r"\b(speculative (decoding|sampling)|draft models?)\b",19 "mixture of experts": r"\b(mixture[- ]of[- ]experts|\bMoE\b|sparsely[- ]gated)\b",20 "reasoning models": r"\b(chain[- ]of[- ]thought|test[- ]time (compute|scaling)|reasoning models?|large reasoning models?|o1-like|R1-like)\b",21 "ai code generation": r"\b(code generation|code LLMs?|program synthesis with (LLMs?|language models)|AI (pair|coding) (programmer|assistants?)|Copilot)\b",22}232425def main():26 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); ap.add_argument("--last-month", default="2026-06"); a = ap.parse_args(); R = a.root27 out = f"{R}/results/emerging"; os.makedirs(out, exist_ok=True)28 con = duckdb.connect(f"{R}/data/processed/wp44.duckdb", read_only=True); con.execute("SET threads TO 16;")29 rows = []; series = {}30 for name, rx in KNOWN.items():31 q = f"""SELECT c.first_month AS m, count(*) n FROM corpus_works c LEFT JOIN oa_abstracts a ON a.work_id = c.id32 WHERE c.eligible AND c.date_precision = 'month' AND c.first_month BETWEEN '2018-01' AND '{a.last_month}'33 AND (regexp_matches(c.title, ?) OR regexp_matches(coalesce(a.abstract, ''), ?)) GROUP BY 1 ORDER BY 1"""34 d = con.execute(q, [rx, rx]).df().set_index("m").n35 months = pd.period_range("2018-01", a.last_month, freq="M").strftime("%Y-%m"); s = d.reindex(months).fillna(0).astype(float); series[name] = s36 v = s.values; det = None; g12 = np.full(len(v), np.nan)37 for t in range(24, len(v)):38 last3 = v[t - 2:t + 1].mean(); trail = v[t - 26:t - 2]; z = (last3 - trail.mean()) / (trail.std() + 1e-6)39 v12 = v[t - 11:t + 1].sum(); v12p = v[t - 23:t - 11].sum(); g12[t] = np.log1p(v12) - np.log1p(v12p)40 if det is None and z > 3 and v12 >= 30 and v12 >= 2 * v12p:41 det = months[t]42 peak = months[int(np.nanargmax(g12))] if np.isfinite(g12).any() else None43 lead = (pd.Period(peak, "M") - pd.Period(det, "M")).n if det and peak else None44 rows.append({"concept": name, "regex": rx, "total_works": int(v.sum()), "first_month_with_5": next((m for m, x in zip(months, np.cumsum(v)) if x >= 5), None),45 "detection_month": det, "peak_growth_month": peak, "lead_months": lead})46 pd.DataFrame(rows).to_csv(f"{out}/known_set_lead_times.csv", index=False); pd.DataFrame(series).to_csv(f"{out}/known_set_monthly.csv")47 print(pd.DataFrame(rows)[["concept", "total_works", "detection_month", "peak_growth_month", "lead_months"]])484950if __name__ == "__main__":51 main()52