WP44: semantic k-NN edges (5.67 M vectors, 56.7 M edges), figure layout fixes
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
49 changed files +1,392 −7
added
data/metadata/semantic_edges_allenai__specter2_base.json
+9 −0
@@ -0,0 +1,9 @@ | |||||
| 1 | +{ | ||||
| 2 | + "model": "allenai/specter2_base", | ||||
| 3 | + "k": 10, | ||||
| 4 | + "min_sim": 0.8, | ||||
| 5 | + "n_vectors": 5668933, | ||||
| 6 | + "n_edges": 56703396, | ||||
| 7 | + "index": "HNSW32 efC80 efS64 inner-product", | ||||
| 8 | + "seconds": 523 | ||||
| 9 | +} | ||||
| \ No newline at end of file | |||||
modified
figures/fig_area_flow_matrix.pdf
+0 −0
Binary file not shown.
modified
figures/fig_area_flow_matrix.png
+0 −0
Binary file not shown.
modified
figures/fig_area_shares.pdf
+0 −0
Binary file not shown.
modified
figures/fig_breaks_by_category.pdf
+0 −0
Binary file not shown.
modified
figures/fig_category_growth.pdf
+0 −0
Binary file not shown.
modified
figures/fig_category_its_panels.pdf
+0 −0
Binary file not shown.
modified
figures/fig_citation_inequality.pdf
+0 −0
Binary file not shown.
modified
figures/fig_cited_age.pdf
+0 −0
Binary file not shown.
modified
figures/fig_collaboration.pdf
+0 −0
Binary file not shown.
modified
figures/fig_country_shares.pdf
+0 −0
Binary file not shown.
modified
figures/fig_cross_area.pdf
+0 −0
Binary file not shown.
modified
figures/fig_cross_area.png
+0 −0
Binary file not shown.
modified
figures/fig_emerging_metrics.pdf
+0 −0
Binary file not shown.
modified
figures/fig_field_inflows.pdf
+0 −0
Binary file not shown.
modified
figures/fig_its_total.pdf
+0 −0
Binary file not shown.
modified
figures/fig_known_set.pdf
+0 −0
Binary file not shown.
modified
figures/fig_llm_cooccurrence.pdf
+0 −0
Binary file not shown.
modified
figures/fig_placebo_total.pdf
+0 −0
Binary file not shown.
modified
figures/fig_prisma_flow.pdf
+0 −0
Binary file not shown.
modified
figures/fig_types_openness.pdf
+0 −0
Binary file not shown.
modified
figures/fig_venue_concentration.pdf
+0 −0
Binary file not shown.
modified
figures/fig_volume_monthly_raw.pdf
+0 −0
Binary file not shown.
modified
figures/fig_volume_yearly.pdf
+0 −0
Binary file not shown.
modified
figures/make_figures.py
+6 −6
@@ -271,7 +271,7 @@ def fig_flow_matrix(R, out): | |||||
| 271 | p = f"{R}/results/graph/area_flows_yearly.csv" | 271 | p = f"{R}/results/graph/area_flows_yearly.csv" | ||
| 272 | if not os.path.exists(p): | 272 | if not os.path.exists(p): | ||
| 273 | return | 273 | return | ||
| 274 | − d = pd.read_csv(p); fig, axes = plt.subplots(1, 2, figsize=(7.4, 3.6)) | 274 | + d = pd.read_csv(p); fig, axes = plt.subplots(1, 2, figsize=(7.6, 4.6)) | ||
| 275 | for ax, (lo, hi), title in [(axes[0], (2018, 2022), "Baseline 2018–2022"), (axes[1], (2023, 2026), "Post 2023–2026")]: | 275 | for ax, (lo, hi), title in [(axes[0], (2018, 2022), "Baseline 2018–2022"), (axes[1], (2023, 2026), "Post 2023–2026")]: | ||
| 276 | g = d[(d.citing_year >= lo) & (d.citing_year <= hi)].groupby(["citing_area", "cited_area"]).n.sum().unstack(fill_value=0) | 276 | g = d[(d.citing_year >= lo) & (d.citing_year <= hi)].groupby(["citing_area", "cited_area"]).n.sum().unstack(fill_value=0) | ||
| 277 | g = g.reindex(index=sorted(set(g.index) | set(g.columns)), columns=sorted(set(g.index) | set(g.columns)), fill_value=0); rs = g.div(g.sum(1), axis=0) * 100 | 277 | g = g.reindex(index=sorted(set(g.index) | set(g.columns)), columns=sorted(set(g.index) | set(g.columns)), fill_value=0); rs = g.div(g.sum(1), axis=0) * 100 | ||
@@ -281,7 +281,7 @@ def fig_flow_matrix(R, out): | |||||
| 281 | for j in range(len(rs.columns)): | 281 | for j in range(len(rs.columns)): | ||
| 282 | v = rs.values[i, j] | 282 | v = rs.values[i, j] | ||
| 283 | if v >= 5: ax.text(j, i, f"{v:.0f}", ha="center", va="center", fontsize=5.5, color="white" if v > 0.6 * rs.values.max() else style.INK) | 283 | if v >= 5: ax.text(j, i, f"{v:.0f}", ha="center", va="center", fontsize=5.5, color="white" if v > 0.6 * rs.values.max() else style.INK) | ||
| 284 | − fig.text(0.5, -0.02, "Row shares (%): references from the citing area (rows) to the cited area (columns), internal citations only", ha="center", fontsize=7, color=style.INK2) | 284 | + fig.subplots_adjust(wspace=0.55, bottom=0.3); fig.text(0.5, 0.01, "Row shares (%): references from the citing area (rows) to the cited area (columns), internal citations only", ha="center", fontsize=7, color=style.INK2) | ||
| 285 | style.save(fig, f"{out}/fig_area_flow_matrix.pdf"); reg("fig_area_flow_matrix", "Citation-flow matrices between areas (row shares) for the baseline and post periods.", [p]) | 285 | style.save(fig, f"{out}/fig_area_flow_matrix.pdf"); reg("fig_area_flow_matrix", "Citation-flow matrices between areas (row shares) for the baseline and post periods.", [p]) | ||
| 286 | 286 | ||||
| 287 | 287 | ||||
@@ -289,14 +289,14 @@ def fig_cross_area(R, out): | |||||
| 289 | p = f"{R}/results/graph/cross_area_share_yearly.csv"; q = f"{R}/results/graph/modularity_yearly.csv" | 289 | p = f"{R}/results/graph/cross_area_share_yearly.csv"; q = f"{R}/results/graph/modularity_yearly.csv" | ||
| 290 | if not os.path.exists(p): | 290 | if not os.path.exists(p): | ||
| 291 | return | 291 | return | ||
| 292 | − d = pd.read_csv(p); fig, axes = plt.subplots(1, 2, figsize=(7.4, 3.2)) | 292 | + d = pd.read_csv(p); fig, axes = plt.subplots(1, 2, figsize=(7.6, 3.4)) | ||
| 293 | for i, a in enumerate(sorted(d.citing_area.unique())): | 293 | for i, a in enumerate(sorted(d.citing_area.unique())): | ||
| 294 | g = d[d.citing_area == a]; axes[0].plot(g.citing_year, 100 * g.share_cross, marker="o", ms=3, color=style.CATEGORICAL[i % 8], label=a, lw=1.1) | 294 | g = d[d.citing_area == a]; axes[0].plot(g.citing_year, 100 * g.share_cross, marker="o", ms=3, color=style.CATEGORICAL[i % 8], label=a, lw=1.1) | ||
| 295 | − axes[0].set_title("Share of references leaving the citing area (%)", loc="left"); axes[0].legend(fontsize=6, ncol=2) | 295 | + axes[0].set_title("References leaving the citing area (%)", loc="left", fontsize=8); axes[0].legend(fontsize=5.5, ncol=2) | ||
| 296 | if os.path.exists(q): | 296 | if os.path.exists(q): | ||
| 297 | m = pd.read_csv(q); axes[1].plot(m.year, m.Q_areas, marker="o", color=style.CATEGORICAL[0], label="Modularity Q (areas)"); axes[1].plot(m.year, m.Q_categories, marker="o", color=style.CATEGORICAL[1], label="Modularity Q (categories)"); axes[1].plot(m.year, m.self_share, marker="o", color=style.CATEGORICAL[2], label="Within-category share") | 297 | m = pd.read_csv(q); axes[1].plot(m.year, m.Q_areas, marker="o", color=style.CATEGORICAL[0], label="Modularity Q (areas)"); axes[1].plot(m.year, m.Q_categories, marker="o", color=style.CATEGORICAL[1], label="Modularity Q (categories)"); axes[1].plot(m.year, m.self_share, marker="o", color=style.CATEGORICAL[2], label="Within-category share") | ||
| 298 | − axes[1].set_title("Modularity of the yearly category citation network", loc="left"); axes[1].legend(fontsize=6) | 298 | + axes[1].set_title("Modularity of the category citation network", loc="left", fontsize=8); axes[1].legend(fontsize=6) | ||
| 299 | − style.save(fig, f"{out}/fig_cross_area.pdf"); reg("fig_cross_area", "Cross-area reference shares by citing area and modularity of the category citation network by year.", [p, q]) | 299 | + fig.subplots_adjust(wspace=0.3); style.save(fig, f"{out}/fig_cross_area.pdf"); reg("fig_cross_area", "Cross-area reference shares by citing area and modularity of the category citation network by year.", [p, q]) | ||
| 300 | 300 | ||||
| 301 | 301 | ||||
| 302 | def fig_emerging(R, out): | 302 | def fig_emerging(R, out): | ||
modified
manuscript/figures/fig_area_flow_matrix.pdf
+0 −0
Binary file not shown.
modified
manuscript/figures/fig_area_shares.pdf
+0 −0
Binary file not shown.
modified
manuscript/figures/fig_breaks_by_category.pdf
+0 −0
Binary file not shown.
modified
manuscript/figures/fig_category_growth.pdf
+0 −0
Binary file not shown.
modified
manuscript/figures/fig_category_its_panels.pdf
+0 −0
Binary file not shown.
modified
manuscript/figures/fig_citation_inequality.pdf
+0 −0
Binary file not shown.
modified
manuscript/figures/fig_cited_age.pdf
+0 −0
Binary file not shown.
modified
manuscript/figures/fig_collaboration.pdf
+0 −0
Binary file not shown.
modified
manuscript/figures/fig_country_shares.pdf
+0 −0
Binary file not shown.
modified
manuscript/figures/fig_cross_area.pdf
+0 −0
Binary file not shown.
modified
manuscript/figures/fig_emerging_metrics.pdf
+0 −0
Binary file not shown.
modified
manuscript/figures/fig_field_inflows.pdf
+0 −0
Binary file not shown.
modified
manuscript/figures/fig_its_total.pdf
+0 −0
Binary file not shown.
modified
manuscript/figures/fig_known_set.pdf
+0 −0
Binary file not shown.
modified
manuscript/figures/fig_llm_cooccurrence.pdf
+0 −0
Binary file not shown.
modified
manuscript/figures/fig_placebo_total.pdf
+0 −0
Binary file not shown.
modified
manuscript/figures/fig_prisma_flow.pdf
+0 −0
Binary file not shown.
modified
manuscript/figures/fig_types_openness.pdf
+0 −0
Binary file not shown.
modified
manuscript/figures/fig_venue_concentration.pdf
+0 −0
Binary file not shown.
modified
manuscript/figures/fig_volume_monthly_raw.pdf
+0 −0
Binary file not shown.
modified
manuscript/figures/fig_volume_yearly.pdf
+0 −0
Binary file not shown.
modified
manuscript/sections/part05_ai.tex
+394 −0
@@ -568,3 +568,397 @@ intellectual map would conclude that generative AI is a topic in education | |||||
| 568 | research; the correct conclusion is that field normalisation inside a | 568 | research; the correct conclusion is that field normalisation inside a | ||
| 569 | multi-label category measures discourse, and that the measurement is | 569 | multi-label category measures discourse, and that the measurement is | ||
| 570 | informative about diffusion rather than about invention. | 570 | informative about diffusion rather than about invention. | ||
| 571 | + | ||||
| 572 | +\section{AI agents and autonomous systems} | ||||
| 573 | +\label{sec:ai-agents} | ||||
| 574 | + | ||||
| 575 | +\subsection{From tool use to agents} | ||||
| 576 | + | ||||
| 577 | +An LLM-based agent, in the definition shared by the surveys of the window | ||||
| 578 | +\citep{wang2024agentsurvey, xi2023rise, guo2024multiagent, | ||||
| 579 | +cheng2026agentsdefinitions}, is a language model wrapped in a loop of | ||||
| 580 | +perception, planning, memory and action, in which the model's outputs are | ||||
| 581 | +parsed into calls to tools or to other agents and the results are fed back as | ||||
| 582 | +context. The architectures that fixed the vocabulary are generative agents in | ||||
| 583 | +a simulated town \citep{park2023generativeagents}, a peer-reviewed UIST paper; | ||||
| 584 | +the multi-agent conversation framework AutoGen \citep{wu2023autogen}; virtual | ||||
| 585 | +context management in MemGPT \citep{packer2023memgpt}; and the ``LLM as | ||||
| 586 | +operating system'' proposals \citep{mei2024aios, ge2023llmos}, all of the | ||||
| 587 | +latter preprints. The memory component is surveyed by | ||||
| 588 | +\citet{zhang2025agentmemory}, whose abstract calls self-evolution through | ||||
| 589 | +agent--environment interaction the feature that distinguishes agents from the | ||||
| 590 | +models they wrap. Evaluation lags design. The benchmarks that the community | ||||
| 591 | +uses---SWE-bench for repository-level issue resolution | ||||
| 592 | +\citep{jimenez2024swebench}, WebArena for web tasks \citep{zhou2023webarena}, | ||||
| 593 | +AgentBench across environments \citep{liu2023agentbench}---are preprints, and | ||||
| 594 | +the only peer-reviewed agent benchmark in the landmark set is a clinical | ||||
| 595 | +one, an evaluation of an open-source and a proprietary agent system on three | ||||
| 596 | +families of medical decision tasks \citep{liu2026clinicalagents}. % [E-32] | ||||
| 597 | +The detector flags \emph{LLM agents} in \KnownLlmAgentsDetectionMonth{}, | ||||
| 598 | +\KnownLlmAgentsLeadMonths{} months before peak growth in | ||||
| 599 | +\KnownLlmAgentsPeakGrowthMonth{}, on \KnownLlmAgentsTotalWorks{} matching | ||||
| 600 | +works; the concept arrived and peaked within a year. | ||||
| 601 | + | ||||
| 602 | +The strongest peer-reviewed evidence for agents is scientific. The Virtual Lab | ||||
| 603 | +\citep{swanson2025virtuallab}, a Nature paper whose record carries no abstract | ||||
| 604 | +in the corpus file, reports agents designing SARS-CoV-2 nanobodies that were | ||||
| 605 | +validated experimentally; Robin \citep{ghareeb2026robin} is described in its | ||||
| 606 | +abstract as the first multi-agent system automating both hypothesis generation | ||||
| 607 | +and data analysis in experimental biology; DIVE \citep{zhang2026dive} extracts | ||||
| 608 | +experimental data from the figures of the literature for hydrogen-storage | ||||
| 609 | +materials; AgentSociety \citep{piao2026agentsociety} simulates large | ||||
| 610 | +populations of LLM-driven generative agents for social science. % [E-33] | ||||
| 611 | +These are single demonstrations with experimental or observational | ||||
| 612 | +validation, which is more than the ``AI scientist'' preprints | ||||
| 613 | +\citep{lu2024aiscientist, si2024novelideas} offer; whether autonomous research | ||||
| 614 | +agents produce validated discoveries at a rate that justifies their cost is a | ||||
| 615 | +question for preregistered multi-laboratory trials, not for case reports, and | ||||
| 616 | +it is listed in Part~X. | ||||
| 617 | + | ||||
| 618 | +\subsection{``Agentic AI'': a definitional literature} | ||||
| 619 | + | ||||
| 620 | +The landmark set of the category contains six papers from 2025--2026 whose | ||||
| 621 | +purpose is to define agentic AI and distinguish it from AI agents, generative | ||||
| 622 | +AI and classical multi-agent systems \citep{acharya2025agenticsurvey, | ||||
| 623 | +sapkota2026agentsvsagentic, bandi2025agenticreview, abouali2025agenticparadigms, | ||||
| 624 | +allmendinger2026multiagentai, collaco2026agentichealthcare}, most of them | ||||
| 625 | +reviews of reviews, one of them a PRISMA-based review of ninety studies that | ||||
| 626 | +proposes a symbolic-versus-neural dual-paradigm framework, and one benchmark. | ||||
| 627 | +% [E-34] | ||||
| 628 | +A literature that produces six taxonomies and one benchmark in eighteen | ||||
| 629 | +months is a literature whose object is not yet stabilised, and the question | ||||
| 630 | +of whether ``agentic AI'' denotes a distinct class of systems or a relabelling | ||||
| 631 | +of tool-augmented LLM pipelines is treated here as open. The trust, risk and | ||||
| 632 | +security management framework for multi-agent LLM systems | ||||
| 633 | +\citep{raza2026trism} belongs to the same genre: a structure without an | ||||
| 634 | +empirical test. | ||||
| 635 | + | ||||
| 636 | +The corpus series of the category must be read with its composition in mind. | ||||
| 637 | +Four of its landmark records are control-theoretic papers on consensus and | ||||
| 638 | +formation control of nonlinear multi-agent systems, and its pre-2022 volume | ||||
| 639 | +is dominated by classical multi-agent systems, distributed artificial | ||||
| 640 | +intelligence and robotics, which explains why the category directs a large | ||||
| 641 | +and growing share of its references to robotics (Table~\ref{tab:ai_flows}). | ||||
| 642 | +The negative baseline growth rate, the negative level change at the reference | ||||
| 643 | +month and the second Bai--Perron break of 2024 (Table~\ref{tab:ai_breaks}) | ||||
| 644 | +are the signature of a category in which an older literature was flat or | ||||
| 645 | +declining while a new one, LLM agents, grew from a small base; the level | ||||
| 646 | +effect at the o1 date, \ItsSeriesCatAiAgentsItsOOneLevelChange{} | ||||
| 647 | +($p = \ItsSeriesCatAiAgentsItsOOneLevelP$), is where the new literature | ||||
| 648 | +becomes visible in the aggregate. % [E-35] | ||||
| 649 | +The explanation is inferential; separating the two literatures would require a | ||||
| 650 | +sub-classification that the taxonomy does not provide. Embodied agents, from | ||||
| 651 | +vision-language-action models \citep{brohan2023rt2, kim2024openvla, | ||||
| 652 | +black2024pi0} to the driving model in the landmark set, are classified under | ||||
| 653 | +robotics and are reviewed there. | ||||
| 654 | + | ||||
| 655 | +\section{Reinforcement learning} | ||||
| 656 | +\label{sec:ai-rl} | ||||
| 657 | + | ||||
| 658 | +The reinforcement-learning landmarks are, almost without exception, the canon | ||||
| 659 | +of the baseline period. Rainbow's combination of six extensions to deep | ||||
| 660 | +Q-learning with an ablation of each \citep{hessel2018rainbow}; the | ||||
| 661 | +maximum-entropy actor--critic \citep{haarnoja2018sac}, whose record is the | ||||
| 662 | +arXiv deposit; clipped double-Q learning and delayed policy updates against | ||||
| 663 | +overestimation \citep{fujimoto2018td3}; self-play without domain knowledge | ||||
| 664 | +beyond the rules in AlphaZero \citep{silver2018alphazero}; grandmaster-level | ||||
| 665 | +StarCraft~II \citep{vinyals2019alphastar}, a Nature record without an abstract | ||||
| 666 | +in the file; the finding that proximal policy optimisation is, in the words of | ||||
| 667 | +its abstract, surprisingly strong in cooperative multi-agent benchmarks | ||||
| 668 | +\citep{yu2021mappo}; the large-scale study of curiosity-driven learning without | ||||
| 669 | +extrinsic rewards \citep{burda2018curiosity}; and the reproducibility warning | ||||
| 670 | +of \citet{henderson2018drlmatters}, whose abstract notes that non-determinism | ||||
| 671 | +and intrinsic variance make reported results hard to interpret. The theoretical | ||||
| 672 | +side is represented by an overview of multi-agent reinforcement learning | ||||
| 673 | +\citep{zhang2021marloverview} and a survey of the synthesis of model-predictive | ||||
| 674 | +control and reinforcement learning (record 7128436456 of the landmark file). | ||||
| 675 | +The single post-2022 method landmark is DreamerV3 \citep{hafner2025dreamerv3}, | ||||
| 676 | +published in Nature, whose abstract reports a world-model algorithm that | ||||
| 677 | +outperforms specialised methods across more than 150 tasks with a single | ||||
| 678 | +configuration. % [E-36] | ||||
| 679 | +Several records are misdated---a 2008 inverse-reinforcement-learning paper | ||||
| 680 | +and a 2007 Bayesian one appear as 2018 and 2025 works---and four are | ||||
| 681 | +control-theory papers; the category's citation top is therefore a mixture of | ||||
| 682 | +classics, control theory and one new result. | ||||
| 683 | + | ||||
| 684 | +The corpus places reinforcement learning among the modestly accelerating | ||||
| 685 | +categories: a post-period growth rate only slightly above the baseline | ||||
| 686 | +(Table~\ref{tab:ai_growth}), a slope change at the reference month of | ||||
| 687 | +\ItsSeriesCatReinforcementLearningItsChatgptSlopeChange{} log points per | ||||
| 688 | +month after a negative level change, and a citation profile that moves towards | ||||
| 689 | +robotics. % [E-15] | ||||
| 690 | +It would be a mistake to read this as the field's eclipse. Reinforcement | ||||
| 691 | +learning's most consequential role in the window---reward modelling and policy | ||||
| 692 | +optimisation for language models, from PPO through DPO to the group-relative | ||||
| 693 | +method of DeepSeek-R1---is invisible in the category because the papers that | ||||
| 694 | +carry it are classified as LLM work, and the flow of references from the RL | ||||
| 695 | +category to the LLM category stays below one per cent. Any statement that | ||||
| 696 | +reinforcement learning declined after 2022 would be an artefact of the | ||||
| 697 | +taxonomy, and the correct statement is that its methods migrated into another | ||||
| 698 | +category's literature. % [E-37] | ||||
| 699 | + | ||||
| 700 | +\section{Machine-learning theory, architectures and efficiency} | ||||
| 701 | +\label{sec:ai-theory} | ||||
| 702 | + | ||||
| 703 | +\subsection{What ``machine-learning theory'' contains in the corpus} | ||||
| 704 | + | ||||
| 705 | +The category named machine-learning theory does not, in its citation-ranked | ||||
| 706 | +top, contain a learning-theory result from the window. Its landmarks are | ||||
| 707 | +textbooks and textbook chapters, the case for the Matthews correlation | ||||
| 708 | +coefficient over accuracy and F$_1$ on imbalanced binary problems | ||||
| 709 | +\citep{chicco2020mcc}, the argument that high-stakes decisions should use | ||||
| 710 | +inherently interpretable models rather than post-hoc explanations of black | ||||
| 711 | +boxes \citep{rudin2019stopexplaining}, the conceptual analysis of | ||||
| 712 | +interpretability \citep{lipton2018mythos}, exact Shapley attributions for tree | ||||
| 713 | +ensembles \citep{lundberg2020treeexplainer}, the review of stochastic-gradient | ||||
| 714 | +optimisation for large-scale learning \citep{bottou2018optimization}, a | ||||
| 715 | +transfer-learning survey \citep{zhuang2021transfersurvey}, a reprinted chapter | ||||
| 716 | +by Verma and Pearl on the equivalence of causal models, and the misdated | ||||
| 717 | +bandit papers of Section~\ref{sec:ai-landmarks}. % [E-38] | ||||
| 718 | +The category as the rules define it is general machine learning, and what the | ||||
| 719 | +citation ranking rewards within it is pedagogy and measurement convention. | ||||
| 720 | +Theoretical work on in-context learning, on the expressivity of Transformers | ||||
| 721 | +or on the statistical basis of scaling laws exists inside the window, but it is | ||||
| 722 | +classified under LLM or deep-learning architectures, or is cited too little | ||||
| 723 | +within thirty-six months to surface. The corpus therefore cannot be used to | ||||
| 724 | +say that theory lagged practice in the AI transformation; it can only say that | ||||
| 725 | +the theory which is cited quickly is the theory of evaluation and | ||||
| 726 | +interpretation. | ||||
| 727 | + | ||||
| 728 | +\subsection{Architectures: proposals, refutations and the efficiency turn} | ||||
| 729 | + | ||||
| 730 | +The deep-learning-architectures landmarks are the mirror image: surveys of | ||||
| 731 | +graph neural networks \citep{wu2021gnnsurvey, zhou2020gnnreview} and of | ||||
| 732 | +Transformers for time series \citep{wen2023tstransformers}, the | ||||
| 733 | +long-sequence forecasting Transformer Informer \citep{zhou2021informer}, and | ||||
| 734 | +its refutation---the finding, reported in the abstract of | ||||
| 735 | +\citet{zeng2023transformerstsf}, that embarrassingly simple linear models | ||||
| 736 | +outperform the Transformer-based forecasters on the standard long-term | ||||
| 737 | +benchmarks because the permutation-invariant attention discards the temporal | ||||
| 738 | +ordering that forecasting depends on. The disagreement is still open within | ||||
| 739 | +the field, and the LLM-reprogramming approach to the same task | ||||
| 740 | +\citep{jin2023timellm} reopens it from a third side. % [E-39] | ||||
| 741 | +Mixup \citep{zhang2017mixup}, the convex-combination regulariser, is in the | ||||
| 742 | +set with a wrong year; the tabular foundation model \citep{hollmann2025tabpfn} | ||||
| 743 | +is its one peer-reviewed post-2022 architectural result. The two architectural | ||||
| 744 | +proposals of the window that the set contains are both unreviewed: Mamba | ||||
| 745 | +\citep{gu2023mamba}, whose abstract identifies the inability of earlier | ||||
| 746 | +subquadratic models to perform content-based reasoning as their weakness and | ||||
| 747 | +makes the state-space parameters input-dependent to obtain linear-time | ||||
| 748 | +sequence modelling, and Kolmogorov--Arnold networks \citep{liu2024kan}, which | ||||
| 749 | +replace fixed node activations by learnable spline functions on edges and | ||||
| 750 | +claim accuracy and interpretability advantages over multilayer perceptrons on | ||||
| 751 | +small scientific problems. Neither claim has been tested at language-model | ||||
| 752 | +scale in a peer-reviewed venue within the corpus. | ||||
| 753 | + | ||||
| 754 | +Where architecture research demonstrably moved is efficiency. Exact attention | ||||
| 755 | +with IO-aware tiling \citep{dao2022flashattention}; sparse mixture of experts | ||||
| 756 | +at trillion-parameter scale \citep{fedus2022switch}; eight-bit matrix | ||||
| 757 | +multiplication with outlier decomposition \citep{dettmers2022int8} and | ||||
| 758 | +post-training weight quantisation to three or four bits \citep{frantar2023gptq}, | ||||
| 759 | +with the quantisation survey in the landmark set \citep{gholami2022quantization}; | ||||
| 760 | +speculative decoding, which samples from a small draft model and verifies in | ||||
| 761 | +parallel with the large one \citep{leviathan2023speculative}; and the | ||||
| 762 | +efficient-LLM and inference surveys \citep{wan2023efficientllm, | ||||
| 763 | +zhou2024inferencesurvey}. The known-set detector gives these topics the | ||||
| 764 | +shortest leads in the area---\KnownStateSpaceSequenceModelsLeadMonths{} months | ||||
| 765 | +for state-space models, \KnownMixtureOfExpertsLeadMonths{} for mixture of | ||||
| 766 | +experts, \KnownSpeculativeDecodingLeadMonths{} for speculative | ||||
| 767 | +decoding---because each was driven by one widely adopted preprint and grew | ||||
| 768 | +from near zero to its peak within a year. % [E-40] | ||||
| 769 | +Deep-learning architectures is also the one AI category without a significant | ||||
| 770 | +post-2022 slope change (Table~\ref{tab:ai_breaks}). Two readings are | ||||
| 771 | +compatible with the counts: that architecture research plateaued once the | ||||
| 772 | +Transformer became the default, or that papers which would have been | ||||
| 773 | +classified as architecture work in 2021 are classified as LLM or | ||||
| 774 | +foundation-model work in 2024 because their vocabulary changed. The | ||||
| 775 | +multi-label counts cannot separate relabelling from decline, and the test | ||||
| 776 | +proposed in Part~X---re-estimating the series on works whose only AI category | ||||
| 777 | +is architectures, with a classifier vocabulary frozen at 2022---has not been | ||||
| 778 | +run. | ||||
| 779 | + | ||||
| 780 | +\section{Evaluation} | ||||
| 781 | +\label{sec:ai-eval} | ||||
| 782 | + | ||||
| 783 | +The evaluation literature of the window is a cycle of benchmark creation, | ||||
| 784 | +saturation and replacement that the landmark set documents from its first | ||||
| 785 | +turn. The SuperGLUE abstract \citep{wang2019superglue} records, already in | ||||
| 786 | +2019, that performance on GLUE \citep{wang2018glue} had ``surpassed the level | ||||
| 787 | +of non-expert humans'' and that a harder suite was needed; the same sentence | ||||
| 788 | +could be written about MMLU \citep{hendrycks2021mmlu} and BIG-bench | ||||
| 789 | +\citep{srivastava2022bigbench} by 2024. % [E-41] | ||||
| 790 | +Three responses coexist. Holistic evaluation across scenarios and metrics | ||||
| 791 | +\citep{liang2022helm} multiplies the axes; crowd-sourced pairwise human | ||||
| 792 | +preference in an arena \citep{chiang2024arena}, a preprint, replaces fixed | ||||
| 793 | +test sets by an Elo-style ranking; and LLM-as-judge \citep{zheng2023judge}, | ||||
| 794 | +also a preprint, replaces the human by a model, with the agreement rates | ||||
| 795 | +reported there as its justification. The QLoRA abstract's ``99.3 per cent of | ||||
| 796 | +ChatGPT'' is an instance of the third: a number produced by GPT-4 judging | ||||
| 797 | +Vicuna-benchmark answers, whose meaning depends on the judge's biases, which | ||||
| 798 | +the LLM-as-judge paper itself catalogues. Against all three stand the | ||||
| 799 | +documented problems of contamination of test sets by training data | ||||
| 800 | +\citep{sainz2023contamination}, the dependence of rankings on the choice of | ||||
| 801 | +tasks \citep{dehghani2021lottery}, benchmark saturation and the | ||||
| 802 | +sociology of benchmark use \citep{ott2022benchmarks, raji2021everything}, the | ||||
| 803 | +case for dynamic, adversarially collected benchmarks \citep{kiela2021dynabench} | ||||
| 804 | +and the earlier demonstration that even a faithfully re-collected test set | ||||
| 805 | +shifts accuracies \citep{recht2019imagenet}. The evaluation survey of | ||||
| 806 | +\citet{chang2024evalsurvey} maps the space. | ||||
| 807 | + | ||||
| 808 | +The corpus adds two facts. The landmark set contains four benchmark | ||||
| 809 | +contributions in 234---SuperGLUE, LAION-5B, MultiMedQA and the clinical agent | ||||
| 810 | +benchmark---so that evaluation work is under-rewarded by thirty-six-month | ||||
| 811 | +citation impact relative to the role the field assigns it, a known property | ||||
| 812 | +of citation measures for infrastructure. % [E-05] | ||||
| 813 | +And no landmark abstract reports a contamination-controlled evaluation: | ||||
| 814 | +the models that the abstracts compare are compared on benchmarks whose | ||||
| 815 | +overlap with their training data is not stated. For a mapping study this is | ||||
| 816 | +the central methodological caveat of the whole area. Every ``state of the art'' | ||||
| 817 | +claim read in an abstract of this part is a claim about a benchmark whose | ||||
| 818 | +validity for the model in question is unknown, and the manuscript's | ||||
| 819 | +distinction between genuine breakthroughs and incremental benchmark | ||||
| 820 | +improvements (RQ7) cannot be drawn from abstracts alone; Part~IX draws it, to | ||||
| 821 | +the extent that it can be drawn, from the citation and adoption structure | ||||
| 822 | +instead. | ||||
| 823 | + | ||||
| 824 | +\section{Safety, alignment, fairness and societal evidence} | ||||
| 825 | +\label{sec:ai-safety} | ||||
| 826 | + | ||||
| 827 | +\subsection{Risk framing} | ||||
| 828 | + | ||||
| 829 | +The safety literature entered the window with a concrete-problems agenda | ||||
| 830 | +\citep{amodei2016concrete, hendrycks2021unsolved} and a critique of scale | ||||
| 831 | +\citep{bender2021parrots}, and left it with a catalogue. The AI Risk | ||||
| 832 | +Repository, published in \emph{Patterns} and in the landmark set, is | ||||
| 833 | +described in its abstract as a living database of 1,725 risks extracted from | ||||
| 834 | +74 taxonomies and organised by a causal taxonomy of origin and a domain | ||||
| 835 | +taxonomy \citep{slattery2026riskrepository}; the International AI Safety | ||||
| 836 | +Report \citep{bengio2025safetyreport} and the \emph{Science} perspective on | ||||
| 837 | +managing extreme risks \citep{bengio2024extremerisks} are the consensus | ||||
| 838 | +documents, and the alignment survey of \citet{ji2023alignment} and the | ||||
| 839 | +foundational-challenges agenda of \citet{anwar2024foundational}, both | ||||
| 840 | +preprints, the research agendas. % [E-42] | ||||
| 841 | +Catastrophic-risk framing \citep{hendrycks2023catastrophic, | ||||
| 842 | +shevlane2023extremerisks}, both preprints, and the ``stochastic parrots'' | ||||
| 843 | +critique disagree about whether scale produces dangerous capabilities at all; | ||||
| 844 | +the corpus shows the AI-safety category re-orienting towards LLM work, which | ||||
| 845 | +it cited at a negligible rate in 2019 and at a tenth of its references in | ||||
| 846 | +2025 (Table~\ref{tab:ai_flows}), but no landmark abstract reports a validated | ||||
| 847 | +measurement of a catastrophic capability, so the disagreement is recorded, not | ||||
| 848 | +resolved. % [E-14] | ||||
| 849 | +The sociology of the category's citation top is the same as that of ML | ||||
| 850 | +theory: it is the pre-LLM ``responsible AI'' literature---fairness | ||||
| 851 | +\citep{mehrabi2021biasfairness}, explainable AI \citep{barredoarrieta2020xai, | ||||
| 852 | +ali2023xaitrust}, explanation as the social sciences understand it | ||||
| 853 | +\citep{miller2019explanation}---rather than the alignment literature of the | ||||
| 854 | +window, which is young and mostly unreviewed. | ||||
| 855 | + | ||||
| 856 | +\subsection{Empirical harms with peer review} | ||||
| 857 | + | ||||
| 858 | +Where the landmark set is strongest is in peer-reviewed empirical harm. The | ||||
| 859 | +data-poisoning threat assessment in \emph{Nature Medicine} reports, in its | ||||
| 860 | +abstract, that replacing 0.001 per cent of the tokens of a common pretraining | ||||
| 861 | +corpus with medical misinformation produces models more likely to propagate | ||||
| 862 | +medical errors \citep{alber2025poisoning}. A physician-led red-teaming of four | ||||
| 863 | +public chatbots on 222 patient-posed primary-care questions found | ||||
| 864 | +statistically significant differences in the safety of their answers | ||||
| 865 | +\citep{draelos2026unsafeanswers}. Experiments on current models as | ||||
| 866 | +replacement therapists report expressions of stigma and inappropriate | ||||
| 867 | +responses contrary to clinical guidelines \citep{moore2025stigma}. % [E-43] | ||||
| 868 | +These three, together with the jailbreak results of Section~\ref{sec:ai-align}, | ||||
| 869 | +are what the corpus can offer as established, replicable harm; the | ||||
| 870 | +over-reliance review \citep{zhai2024overreliance}, the healthcare bias | ||||
| 871 | +review \citep{hasanzadeh2025biashealthcare} and the governance frameworks | ||||
| 872 | +\citep{papagiannidis2025raigovernance} are syntheses and proposals. Energy is | ||||
| 873 | +the one harm with a measurement standard: the inference-cost comparison in | ||||
| 874 | +the landmark set reports, in its abstract, the first systematic comparison of | ||||
| 875 | +task-specific and general-purpose models and finds the latter more expensive | ||||
| 876 | +by a wide margin \citep{luccioni2024powerhungry}, in line with the | ||||
| 877 | +electricity-footprint estimates of \citet{devries2023energy}. % [E-44] | ||||
| 878 | + | ||||
| 879 | +\subsection{Labour, adoption and education} | ||||
| 880 | + | ||||
| 881 | +The social-science records of the set carry the window's evidence on who uses | ||||
| 882 | +the models and to what effect. Occupational exposure to language models is | ||||
| 883 | +estimated from task decompositions in \citet{eloundou2024gpts}, published in | ||||
| 884 | +\emph{Science}, whose landmark record, as noted, carries the wrong abstract; | ||||
| 885 | +the preregistered field experiment with 758 consultants, whose | ||||
| 886 | +\emph{Organization Science} version is in the set, reports that assistance | ||||
| 887 | +improved performance on some tasks and worsened it on others of similar | ||||
| 888 | +apparent difficulty, the ``jagged frontier'' \citep{dellacqua2023jagged}; and | ||||
| 889 | +the adoption surveys report use by nearly half of United States adults within | ||||
| 890 | +two years of the first product \citep{bick2026rapidadoption}. % [E-45] | ||||
| 891 | +In education, the category where the ChatGPT commentary concentrated, the | ||||
| 892 | +abstracts move from perception surveys \citep{chan2023students, | ||||
| 893 | +kasneci2023chatgptgood} to policy analysis \citep{mcdonald2025hepolicies, | ||||
| 894 | +chan2023policyframework} and to one preregistered experiment reporting harm | ||||
| 895 | +to unassisted performance \citep{bastani2024harmlearning}, a preprint. The | ||||
| 896 | +direction of the effect of generative-AI use on learning is thus contested | ||||
| 897 | +between survey evidence of perceived benefit and experimental evidence of | ||||
| 898 | +harm, and the manuscript lists it among the open problems rather than taking | ||||
| 899 | +a side. | ||||
| 900 | + | ||||
| 901 | +\section{Synthesis: established, contested, speculative} | ||||
| 902 | +\label{sec:ai-synthesis} | ||||
| 903 | + | ||||
| 904 | +Four reconciliations summarise what this part can say and what it cannot. | ||||
| 905 | + | ||||
| 906 | +\emph{Growth versus discontinuity.} The nine AI categories accelerated after | ||||
| 907 | +2022; the acceleration is a slope change that the data-driven procedures date | ||||
| 908 | +to the middle of 2022 or later and that steepened in 2023--2024, not a | ||||
| 909 | +level jump in December 2022 (Tables~\ref{tab:ai_breaks} and | ||||
| 910 | +\ref{tab:ai_altdates}). The design cannot attribute it to one product, and | ||||
| 911 | +sentences of the form ``ChatGPT caused a surge in AI research'' are not | ||||
| 912 | +supported by anything in this manuscript. What is supported is weaker and | ||||
| 913 | +more useful: the trend of the LLM literature changed, the change propagated | ||||
| 914 | +through citation to generative AI, safety and agents within three years, and | ||||
| 915 | +it did not propagate to reinforcement learning, theory or architectures. | ||||
| 916 | +% [E-09--E-16] | ||||
| 917 | + | ||||
| 918 | +\emph{Impact versus invention.} The citation-ranked landmarks of the LLM and | ||||
| 919 | +generative-AI categories are commentary and education papers; the technical | ||||
| 920 | +lineage is reached through arXiv records that the bibliographic linkage | ||||
| 921 | +sometimes attached to the wrong work. A list of the most influential AI papers | ||||
| 922 | +of the window built from this corpus without the screening of | ||||
| 923 | +Section~\ref{sec:ai-landmarks} would be wrong in about one record in five, and | ||||
| 924 | +would rank the first wave of ChatGPT commentary above the papers that | ||||
| 925 | +introduced the models. % [E-02, E-06] | ||||
| 926 | + | ||||
| 927 | +\emph{Taxonomy versus field.} The apparent stagnation of reinforcement | ||||
| 928 | +learning and of deep-learning architectures is at least partly a consequence | ||||
| 929 | +of classification: the methods of the first migrated into LLM post-training, | ||||
| 930 | +and the vocabulary of the second migrated into the LLM and foundation-model | ||||
| 931 | +categories. % [E-37, P-08] | ||||
| 932 | + | ||||
| 933 | +\emph{Evidence tiers.} Twenty-eight of the 234 landmark candidates are | ||||
| 934 | +preprints and seventy-two are dated 2026 with twelve-month citation windows; | ||||
| 935 | +many of the window's defining technical papers---LLaMA, BLIP-2, Mamba, the | ||||
| 936 | +agent benchmarks, the emergent-abilities debate, the catastrophic-risk | ||||
| 937 | +agenda---have not passed peer review. The established results of this part | ||||
| 938 | +are therefore of two kinds: corpus statistics, which are reproducible from | ||||
| 939 | +the files named in the tables, and peer-reviewed empirical findings, of which | ||||
| 940 | +the landmark set contains a modest number, concentrated in medicine | ||||
| 941 | +(foundation models, poisoning, unsafe answers), science (MatterGen, the | ||||
| 942 | +Virtual Lab, Robin, DreamerV3) and the social sciences (adoption, the jagged | ||||
| 943 | +frontier). The contested results---emergence, sparks, pure-RL reasoning, | ||||
| 944 | +linear versus Transformer forecasting, fusion versus text-conditioning, | ||||
| 945 | +agentic AI as a category, the sign of the learning effect---are contested in | ||||
| 946 | +the literature itself. The speculative ones---domain foundation models | ||||
| 947 | +displacing pipelines, autonomous science at scale, trust frameworks for | ||||
| 948 | +multi-agent systems---are speculative because no landmark abstract tests | ||||
| 949 | +them. | ||||
| 950 | + | ||||
| 951 | +The open problems that this reading leaves are collected in | ||||
| 952 | +\texttt{literature\_synthesis/agent\_E/open\_problems.csv} and taken up in | ||||
| 953 | +Part~X: whether capabilities are discontinuous in scale; whether pure-RL | ||||
| 954 | +reasoning generalises beyond verifiable rewards; whether alignment brittleness | ||||
| 955 | +is intrinsic; how to measure hallucination without references; whether | ||||
| 956 | +vision-language models fuse or condition; what ``agentic'' denotes; whether | ||||
| 957 | +autonomous science agents pay; whether architecture research plateaued or was | ||||
| 958 | +relabelled; how to evaluate on benchmarks that may be in the training data; | ||||
| 959 | +whether catastrophic capabilities can be measured before deployment; whether | ||||
| 960 | +domain foundation models generalise across sites; whether Transformers suit | ||||
| 961 | +long-horizon time series; whether generative-AI use helps or harms learning; | ||||
| 962 | +and, for this manuscript's own method, whether citation-based landmark | ||||
| 963 | +identification can be made reliable for a field whose most-cited identifiers | ||||
| 964 | +are the ones most often attached to the wrong record. | ||||
modified
manuscript/sections/part07a_theory.tex
+622 −1
@@ -1 +1,622 @@ | |||||
| 1 | −% part07a_theory — being written by Agent G | 1 | +% part07a_theory.tex — Part VII (theory half): algorithms and data structures, computational complexity, programming | ||
| 2 | +% languages, compilers, formal verification, software engineering. Agent G (Theory Specialist), 2026-11-03. | ||||
| 3 | +% The director merges this file with part07b_security.tex into Part VII. Every corpus number is a macro of | ||||
| 4 | +% tables/numbers.tex (prefix \Thy..., appended by tables/make_numbers.py, Agent G block) or appears in a generated table | ||||
| 5 | +% (tables/make_tables.py, tables_part07a). Sources: results/graph/category_counts_yearly.csv, category_flows_yearly.csv, | ||||
| 6 | +% category_cooccurrence_yearly.csv; results/econometrics/breaks.csv; literature_synthesis/landmarks/{theory,software_pl}_*; | ||||
| 7 | +% literature_synthesis/agent_G/{claims,contributions,open_problems}.csv and review.md. Bibliography: references.bib and | ||||
| 8 | +% references_part07a.bib (65 entries, each resolved by one live lookup; references_verification_part07a.csv). | ||||
| 9 | + | ||||
| 10 | +\section{Scope, evidence and reading rules for the theory half} | ||||
| 11 | +\label{sec:thy-scope} | ||||
| 12 | + | ||||
| 13 | +This half of Part~VII reviews six categories of the taxonomy: algorithms and data structures, computational | ||||
| 14 | +complexity, programming languages, compilers, formal verification and software engineering (the areas | ||||
| 15 | +\emph{theory} and \emph{software\_pl} of the ontology, Section~\ref{sec:p2-ontology}). It has two objects. The | ||||
| 16 | +first is the research itself: what the baseline period established in these categories, what the window added, | ||||
| 17 | +and where the additions are contested. The second is the instrument: the corpus statistics for the six | ||||
| 18 | +categories and, above all, the automatically selected landmark candidates, which in these areas turn out to | ||||
| 19 | +measure the bibliographic record more than the research it records. The two objects cannot be separated, | ||||
| 20 | +because the landmark list is the protocol's channel from corpus to synthesis, and in the theory and software | ||||
| 21 | +areas that channel is obstructed in ways that are themselves findings. | ||||
| 22 | + | ||||
| 23 | +Three reading rules apply. First, no full text was read. Every statement about a work rests on its metadata | ||||
| 24 | +(title, venue, type, year, identifiers) or on the abstract recorded in OpenAlex, and the text says which; where | ||||
| 25 | +an abstract is the source, the phrase ``the abstract reports'' is used deliberately. Second, works available | ||||
| 26 | +only as arXiv postings are preprints and are described as such, whatever their later fate. Third, the | ||||
| 27 | +structural-change statistics are read as the protocol prescribes (Section~\ref{sec:m-structural}): the November | ||||
| 28 | +2022 boundary is a reference date, the placebo distribution is the test, and nothing below attributes a change | ||||
| 29 | +to any single event. The categories are small relative to the AI categories of Part~V (their combined primary | ||||
| 30 | +share in 2025 is under ten per cent of the corpus), so the statistics are those of minorities inside a | ||||
| 31 | +corpus whose growth is set elsewhere. | ||||
| 32 | + | ||||
| 33 | +The section proceeds from the instrument to the field. Section~\ref{sec:thy-landmarks} audits the landmark | ||||
| 34 | +candidates; Section~\ref{sec:thy-statistics} reports the six categories' volume, structural-change and | ||||
| 35 | +composition statistics; Sections~\ref{sec:thy-algorithms} to~\ref{sec:thy-se} review the categories; and | ||||
| 36 | +Section~\ref{sec:thy-synthesis} draws the conclusions that Part~X and Part~XI inherit. | ||||
| 37 | + | ||||
| 38 | +\section{The landmark candidates: an audit} | ||||
| 39 | +\label{sec:thy-landmarks} | ||||
| 40 | + | ||||
| 41 | +The protocol selects landmarks as the works with the highest field-normalised 36-month citation score | ||||
| 42 | +(NCS36, Section~\ref{sec:m-citations}) within each primary category, with a floor of fifteen per category, plus | ||||
| 43 | +the 200 most cited works overall; for 2025--2026 works the 12-month window replaces the 36-month one | ||||
| 44 | +(Section~\ref{sec:data-core}). The theory area contributed \ThyLandTheoryN{} candidates and the software area | ||||
| 45 | +\ThyLandSoftwarePlN{}, twenty-five per category, of the \SelectionLandmarks{} landmarks of the whole corpus. | ||||
| 46 | +Before any of the \ThyAuditN{} can be called a landmark, the list has to be audited, and the audit is the first | ||||
| 47 | +result of this section (Table~\ref{tab:theory_landmark_audit}). Each candidate received one of eight verdicts | ||||
| 48 | +from its metadata and, where available, its abstract: an in-scope research contribution | ||||
| 49 | +(\ThyAuditVerdictInScopeResearch{} candidates), an in-scope survey or report (\ThyAuditVerdictInScopeSurvey{}), | ||||
| 50 | +a misclassified work outside the category and outside the theory and software scope | ||||
| 51 | +(\ThyAuditVerdictMisclassified{}), a thesis (\ThyAuditVerdictDissertation{}), a classic re-deposited under a | ||||
| 52 | +recent record date (\ThyAuditVerdictReprintMisdated{}), a textbook or chapter (\ThyAuditVerdictTextbook{}), a | ||||
| 53 | +metaphor-based metaheuristic (\ThyAuditVerdictMetaheuristic{}) or a duplicate record | ||||
| 54 | +(\ThyAuditVerdictDuplicate{}). Fewer than one candidate in five is a research contribution inside its | ||||
| 55 | +category, and only \ThyAuditInScopeResearchPost{} of those are post-period works. Four mechanisms account | ||||
| 56 | +for the rest. | ||||
| 57 | + | ||||
| 58 | +\input{tables/tab_theory_landmark_audit} | ||||
| 59 | + | ||||
| 60 | +\paragraph{Identifier collisions.} | ||||
| 61 | +\ThyAuditIdCollision{} candidates carry OpenAlex identifiers that, resolved against the live database on | ||||
| 62 | +3~November 2026, return a different work. The record titled ``Exploiting Generative AI to Scale up Intelligent | ||||
| 63 | +Tutoring Systems'', with the DOI of an ITP~2023 paper, resolves to \emph{Attention Is All You Need}; the abstract | ||||
| 64 | +stored for it in the corpus is that of \emph{MizAR~60 for Mizar~50}, which is what the DOI denotes | ||||
| 65 | +\citep{jakubuv2023mizar}; its 36-month citation count in the corpus is that of the Transformer paper, which is | ||||
| 66 | +why it ranks first in the whole software area. The ECOOP~2025 paper on functionality-specific | ||||
| 67 | +vulnerabilities \citep{chen2025apiss} resolves to the graph-convolutional-network preprint; the SEA~2024 | ||||
| 68 | +paper on learned branching for maximum independent set \citep{langedal2024mis} to GraphSAGE; the WADS~2025 | ||||
| 69 | +Kd-tree paper \citep{pham2025kdtree} to the variational-autoencoder preprint; the SAIA~2024 robustness | ||||
| 70 | +framework \citep{awadid2025robust} to the adversarial-training preprint of Madry and co-authors; an | ||||
| 71 | +ESEM~2026 record \citep{naqvi2026bigger} to the Codex paper; and a DX~2024 answer-set-programming paper | ||||
| 72 | +\citep{klampfl2024asp} to an unrelated preprint. In every case the recorded citation window matches the other | ||||
| 73 | +work's count.\footnote{Raw responses in \texttt{literature\_synthesis/agent\_G/verification\_lookups\_raw.txt}. | ||||
| 74 | +The mechanism (identifier merging or reuse between the snapshot of 14~October 2026 and the live database) is | ||||
| 75 | +inferred; the consequence, that the citation windows of these records belong to other works, is observed.} The | ||||
| 76 | +four top-ranked candidates of the software area and the top-ranked candidate of the theory area owe their rank | ||||
| 77 | +to citations that are not theirs. The same check should be run on the other areas' lists before Part~IX uses | ||||
| 78 | +them. | ||||
| 79 | + | ||||
| 80 | +\paragraph{Generic-title chapters.} | ||||
| 81 | +Chapters titled ``Introduction to Algorithms'' (2020, 2021, 2023), ``Particle Swarm Optimization'' (2019), | ||||
| 82 | +``Introduction to Software Testing'' (2021), ``Introduction to Graph Theory'' (2023), and Royce's 1970 paper on | ||||
| 83 | +managing large software developments reprinted in a 2021 anthology, carry hundreds to thousands of citations in | ||||
| 84 | +their windows, and live lookups confirm the counts. The plausible reading, offered as an interpretation, is | ||||
| 85 | +that citations intended for the canonical work are attributed to a same-titled chapter by the indexer's | ||||
| 86 | +reference matching. Whatever the mechanism, a chapter of a 2023 undergraduate textbook is not a landmark of | ||||
| 87 | +2023 research. | ||||
| 88 | + | ||||
| 89 | +\paragraph{Re-deposited classics.} | ||||
| 90 | +\ThyAuditVerdictReprintMisdated{} records are works whose record date is the date of a deposit, reprint or | ||||
| 91 | +re-posting rather than of the work. Dijkstra's 1959 note on two problems in connexion with graphs appears as a | ||||
| 92 | +2022 ACM Books chapter; Turing's 1936 paper as a 2021 MIT Press chapter; the 1994 survey of decidability for | ||||
| 93 | +Petri nets as a 2024 arXiv posting; Lynch and Tuttle's introduction to input/output automata as a 2026 journal | ||||
| 94 | +article \citep{lynch2026ioautomata}; the \emph{egg} equality-saturation paper \citep{willsey2021egg} as a 2026 | ||||
| 95 | +research highlight \citep{willsey2026eggcacm}; doctoral theses of the early 2000s as 2026 repository records; | ||||
| 96 | +three theses of the L4/seL4 verification school as 2022 records; and the 2018 conference paper on online | ||||
| 97 | +algorithms with machine-learned predictions as a 2024 arXiv posting. Because the year field is the deposit year, | ||||
| 98 | +these records enter the post period: \ThyLandTheoryDatedTwentysix{} of the \ThyLandTheoryN{} theory candidates | ||||
| 99 | +and \ThyLandSoftwarePlDatedTwentysix{} of the \ThyLandSoftwarePlN{} software candidates are dated 2026, a year | ||||
| 100 | +of which the corpus holds ten months. | ||||
| 101 | + | ||||
| 102 | +\paragraph{Lexical misclassification and small cells.} | ||||
| 103 | +\ThyAuditVerdictMisclassified{} candidates are outside the theory and software scope altogether. The | ||||
| 104 | +computational-complexity list contains \ThyAuditComputationalComplexityMisclassified{} such works of its | ||||
| 105 | +\ThyAuditComputationalComplexityN{}: technology-acceptance models of mobile learning, PLS-SEM studies of | ||||
| 106 | +small enterprises and of students' well-being, a Routledge volume on complexity theory in the social sciences, | ||||
| 107 | +a review of fringe-projection profilometry, captured by the word ``complexity''. The compilers list contains | ||||
| 108 | +\ThyAuditCompilersMisclassified{}, captured by ``just-in-time'' (adaptive interventions in mental health), | ||||
| 109 | +``compiling'' (a guide to literature reviews) and ``polyhedral'' (power-system dispatch). The | ||||
| 110 | +programming-languages list includes a Nature paper on Calabi--Yau manifolds in black-hole scattering. The | ||||
| 111 | +classifier's per-category precision, estimated on the validation sample (Section~\ref{sec:m-validation}), is | ||||
| 112 | +the right place to quantify this; what the landmark list shows is the interaction between imprecision and | ||||
| 113 | +normalisation. NCS36 divides a work's citations by the expectation for its category, year and type | ||||
| 114 | +(Section~\ref{sec:m-citations}); cells of theses or of non-computing articles are small and lightly cited, so a | ||||
| 115 | +thesis with a handful of citations, or a social-science article with a hundred, obtains a normalised score that | ||||
| 116 | +no STOC paper of the same year can match. \ThyAuditVerdictDissertation{} of the \ThyAuditN{} candidates are | ||||
| 117 | +theses. | ||||
| 118 | + | ||||
| 119 | +What survives is a set of \ThyAuditVerdictInScopeResearch{} research works and \ThyAuditVerdictInScopeSurvey{} | ||||
| 120 | +surveys or reports, listed in Table~\ref{tab:theory_landmarks_discussed}. They are reviewed below with the | ||||
| 121 | +pre-2022 lineage of Part~II and with the window's results that the ranking does not see: the major theoretical | ||||
| 122 | +results of 2022--2025 are absent from the candidate list and enter this review through verified bibliographic | ||||
| 123 | +records, not through the corpus ranking. Open problem~G-P1 of Part~X states the correction: type filters, | ||||
| 124 | +live re-resolution of identifiers and venue priors at the selection stage, evaluated against a | ||||
| 125 | +specialist-labelled sample. | ||||
| 126 | + | ||||
| 127 | +\input{tables/tab_theory_landmarks_discussed} | ||||
| 128 | + | ||||
| 129 | +\section{Volume, structural change and composition} | ||||
| 130 | +\label{sec:thy-statistics} | ||||
| 131 | + | ||||
| 132 | +\subsection{Growth in absolute terms, decline in relative terms} | ||||
| 133 | + | ||||
| 134 | +Table~\ref{tab:theory_counts} gives the yearly counts of works carrying each category (multi-label) and of | ||||
| 135 | +works whose primary category it is; Table~\ref{tab:theory_summary} summarises growth and structural change. | ||||
| 136 | +The multi-label counts grew in every category after 2022. Between 2023 and 2025 the compound annual growth was | ||||
| 137 | +\ThyCagrMultilabelSoftwareEngineeringPostPct\% for software engineering, | ||||
| 138 | +\ThyCagrMultilabelComputationalComplexityPostPct\% for computational complexity, | ||||
| 139 | +\ThyCagrMultilabelCompilersPostPct\% for compilers, \ThyCagrMultilabelAlgorithmsPostPct\% for algorithms, | ||||
| 140 | +\ThyCagrMultilabelFormalVerificationPostPct\% for formal verification and | ||||
| 141 | +\ThyCagrMultilabelProgrammingLanguagesPostPct\% for programming languages, against | ||||
| 142 | +\ThyCagrMultilabelSoftwareEngineeringBaselinePct\%, \ThyCagrMultilabelComputationalComplexityBaselinePct\%, | ||||
| 143 | +\ThyCagrMultilabelCompilersBaselinePct\%, \ThyCagrMultilabelAlgorithmsBaselinePct\%, | ||||
| 144 | +\ThyCagrMultilabelFormalVerificationBaselinePct\% and \ThyCagrMultilabelProgrammingLanguagesBaselinePct\% | ||||
| 145 | +between 2018 and 2022. The acceleration is real but not specific: the corpus-wide multi-label growth rose from | ||||
| 146 | +\ThyCagrMultilabelTotalBaselinePct\% to \ThyCagrMultilabelTotalPostPct\% over the same periods, so the six | ||||
| 147 | +categories accelerated with the corpus rather than within it. Measured by primary-label share, every category | ||||
| 148 | +lost ground between 2018 and 2025: algorithms from \ThySharePrimaryAlgorithmsTwoZeroOneEightPct\% to | ||||
| 149 | +\ThySharePrimaryAlgorithmsTwoZeroTwoFivePct\%, software engineering from | ||||
| 150 | +\ThySharePrimarySoftwareEngineeringTwoZeroOneEightPct\% to \ThySharePrimarySoftwareEngineeringTwoZeroTwoFivePct\%, | ||||
| 151 | +formal verification from \ThySharePrimaryFormalVerificationTwoZeroOneEightPct\% to | ||||
| 152 | +\ThySharePrimaryFormalVerificationTwoZeroTwoFivePct\%, programming languages from | ||||
| 153 | +\ThySharePrimaryProgrammingLanguagesTwoZeroOneEightPct\% to \ThySharePrimaryProgrammingLanguagesTwoZeroTwoFivePct\%, | ||||
| 154 | +computational complexity from \ThySharePrimaryComputationalComplexityTwoZeroOneEightPct\% to | ||||
| 155 | +\ThySharePrimaryComputationalComplexityTwoZeroTwoFivePct\% and compilers from | ||||
| 156 | +\ThySharePrimaryCompilersTwoZeroOneEightPct\% to \ThySharePrimaryCompilersTwoZeroTwoFivePct\%. The theory and | ||||
| 157 | +software categories are growing and shrinking at once: more works, a smaller fraction of a corpus whose growth | ||||
| 158 | +is set by the AI categories of Part~V. | ||||
| 159 | + | ||||
| 160 | +\input{tables/tab_theory_counts} | ||||
| 161 | + | ||||
| 162 | +Two cautions attach to these numbers. The post-period growth rate spans two years and is exposed to the | ||||
| 163 | +indexing lag of the most recent year (Section~\ref{sec:m-truncation}); the primary-label counts for 2025 are | ||||
| 164 | +the last complete ones. And the file \texttt{results/bibliometrics/cagr\_by\_category.csv}, from which | ||||
| 165 | +Table~\ref{tab:cagr} is generated, reports primary counts for these categories about three times smaller than | ||||
| 166 | +those of the graph counts file (algorithms in 2018: \ThyCagrfileAlgorithmsNTwoZeroOneEight{} against | ||||
| 167 | +\ThyCountAlgorithmsTwoZeroOneEightPrimary{}), so it predates the current classification; its growth rates are | ||||
| 168 | +not used here, and the discrepancy is recorded for the reproducibility audit.\footnote{The generator | ||||
| 169 | +\texttt{bibliometrics/04\_diversity\_cagr.py} reads \texttt{category\_counts\_yearly.csv}; re-running it on the | ||||
| 170 | +current counts resolves the discrepancy. Until then, Table~\ref{tab:cagr} and Table~\ref{tab:theory_summary} | ||||
| 171 | +disagree for these six rows.} | ||||
| 172 | + | ||||
| 173 | +\subsection{Structural change at the reference date} | ||||
| 174 | + | ||||
| 175 | +The interrupted-time-series slope change at December 2022 is positive for all six primary-label series and | ||||
| 176 | +significant with HAC standard errors (Section~\ref{sec:m-its}): \ThyBreakAlgorithmsChatgptSlopePct\% per month | ||||
| 177 | +for algorithms, \ThyBreakComputationalComplexityChatgptSlopePct\% for computational complexity, | ||||
| 178 | +\ThyBreakProgrammingLanguagesChatgptSlopePct\% for programming languages, | ||||
| 179 | +\ThyBreakCompilersChatgptSlopePct\% for compilers, \ThyBreakFormalVerificationChatgptSlopePct\% for formal | ||||
| 180 | +verification and \ThyBreakSoftwareEngineeringChatgptSlopePct\% for software engineering. The placebo test | ||||
| 181 | +(Section~\ref{sec:m-placebo}) does not confirm any of them at the five per cent level: the share of | ||||
| 182 | +baseline-period pseudo-events whose slope statistic exceeds the observed one is | ||||
| 183 | +\ThyBreakAlgorithmsPlaceboSlopeP{} for algorithms, \ThyBreakComputationalComplexityPlaceboSlopeP{} for | ||||
| 184 | +computational complexity, \ThyBreakSoftwareEngineeringPlaceboSlopeP{} for software engineering, | ||||
| 185 | +\ThyBreakCompilersPlaceboSlopeP{} for compilers, \ThyBreakFormalVerificationPlaceboSlopeP{} for formal | ||||
| 186 | +verification and \ThyBreakProgrammingLanguagesPlaceboSlopeP{} for programming languages. The two theory | ||||
| 187 | +categories are closest to the threshold; none crosses it. A slope change that pseudo-events of the baseline | ||||
| 188 | +reproduce at these rates is a feature of series that were already accelerating, not of the reference date. | ||||
| 189 | + | ||||
| 190 | +\input{tables/tab_theory_summary} | ||||
| 191 | + | ||||
| 192 | +The level changes point the other way in five categories. At December 2022 the level fell by | ||||
| 193 | +\ThyBreakFormalVerificationChatgptLevelPct\% for formal verification ($p = \ThyBreakFormalVerificationChatgptLevelP$), | ||||
| 194 | +\ThyBreakComputationalComplexityChatgptLevelPct\% for computational complexity | ||||
| 195 | +($p = \ThyBreakComputationalComplexityChatgptLevelP$), \ThyBreakAlgorithmsChatgptLevelPct\% for algorithms | ||||
| 196 | +($p = \ThyBreakAlgorithmsChatgptLevelP$), \ThyBreakProgrammingLanguagesChatgptLevelPct\% for programming | ||||
| 197 | +languages and \ThyBreakCompilersChatgptLevelPct\% for compilers (both not significant); only software | ||||
| 198 | +engineering shows a positive level change (\ThyBreakSoftwareEngineeringChatgptLevelPct\%, | ||||
| 199 | +$p = \ThyBreakSoftwareEngineeringChatgptLevelP$), and it is also the only category with a significant positive | ||||
| 200 | +level change at the GPT-4 date (\ThyBreakSoftwareEngineeringGptFourLevel{}, | ||||
| 201 | +$p = \ThyBreakSoftwareEngineeringGptFourLevelP$; Section~\ref{sec:m-altdates}). The negative level changes | ||||
| 202 | +are at least partly the seasonality of proceedings dates in categories dominated by conferences, and the | ||||
| 203 | +alternative-date estimates for the o1 date (\ThyBreakComputationalComplexityOOneLevel{} for computational | ||||
| 204 | +complexity, $p = \ThyBreakComputationalComplexityOOneLevelP$) show how sensitive a level estimate is to where | ||||
| 205 | +the window is cut. The Bai--Perron procedure (Section~\ref{sec:m-bp}) dates a single break at | ||||
| 206 | +\ThyBreakAlgorithmsBpBreaksBic{} for algorithms, computational complexity and compilers, two breaks for | ||||
| 207 | +programming languages (\ThyBreakProgrammingLanguagesBpBreaksBic{}) and formal verification | ||||
| 208 | +(\ThyBreakFormalVerificationBpBreaksBic{}), and one at \ThyBreakSoftwareEngineeringBpBreaksBic{} for software | ||||
| 209 | +engineering: the data-driven dates fall in spring 2023 for the three categories with a single break, which is | ||||
| 210 | +after the reference date and before the GPT-4 release had been indexed, and they say nothing about cause. | ||||
| 211 | + | ||||
| 212 | +\subsection{Vocabulary moves, citations do not} | ||||
| 213 | + | ||||
| 214 | +Composition is where the window is visible. Table~\ref{tab:theory_llm_cooccurrence} reports, for each | ||||
| 215 | +category and year, the number of works that the multi-label classifier also assigns to the LLM category and | ||||
| 216 | +their share of the category. Between 2022 and 2025 that share rose from | ||||
| 217 | +\ThyCoocSoftwareEngineeringLlmTwoZeroTwoTwoPct\% to \ThyCoocSoftwareEngineeringLlmTwoZeroTwoFivePct\% in | ||||
| 218 | +software engineering, from \ThyCoocProgrammingLanguagesLlmTwoZeroTwoTwoPct\% to | ||||
| 219 | +\ThyCoocProgrammingLanguagesLlmTwoZeroTwoFivePct\% in programming languages, from | ||||
| 220 | +\ThyCoocCompilersLlmTwoZeroTwoTwoPct\% to \ThyCoocCompilersLlmTwoZeroTwoFivePct\% in compilers, from | ||||
| 221 | +\ThyCoocFormalVerificationLlmTwoZeroTwoTwoPct\% to \ThyCoocFormalVerificationLlmTwoZeroTwoFivePct\% in formal | ||||
| 222 | +verification, and from \ThyCoocAlgorithmsLlmTwoZeroTwoTwoPct\% to \ThyCoocAlgorithmsLlmTwoZeroTwoFivePct\% in | ||||
| 223 | +both algorithms and computational complexity. The gradient runs from the code-facing categories to the | ||||
| 224 | +mathematical ones, and it is a gradient of vocabulary: co-classification is a lexical and topical signal | ||||
| 225 | +(Section~\ref{sec:m-classification}), not a measure of method adoption. | ||||
| 226 | + | ||||
| 227 | +\input{tables/tab_theory_llm_cooccurrence} | ||||
| 228 | + | ||||
| 229 | +Citation flows are more conservative. In the post period, software-engineering works direct | ||||
| 230 | +\ThyFlowSoftwareEngineeringToLlmPostPct\% of their categorised references to works of the LLM category, | ||||
| 231 | +computational complexity \ThyFlowComputationalComplexityToLlmPostPct\%, compilers | ||||
| 232 | +\ThyFlowCompilersToLlmPostPct\%, programming languages \ThyFlowProgrammingLanguagesToLlmPostPct\%, formal | ||||
| 233 | +verification \ThyFlowFormalVerificationToLlmPostPct\% and algorithms \ThyFlowAlgorithmsToLlmPostPct\%; the | ||||
| 234 | +within-category shares barely moved (algorithms \ThyFlowAlgorithmsToAlgorithmsBaselinePct\% to | ||||
| 235 | +\ThyFlowAlgorithmsToAlgorithmsPostPct\%, software engineering | ||||
| 236 | +\ThyFlowSoftwareEngineeringToSoftwareEngineeringBaselinePct\% to | ||||
| 237 | +\ThyFlowSoftwareEngineeringToSoftwareEngineeringPostPct\%, formal verification | ||||
| 238 | +\ThyFlowFormalVerificationToFormalVerificationBaselinePct\% to | ||||
| 239 | +\ThyFlowFormalVerificationToFormalVerificationPostPct\%).\footnote{Flows are aggregated from | ||||
| 240 | +\texttt{results/graph/category\_flows\_yearly.csv} over citing years 2018--2022 (baseline) and 2023--2026 | ||||
| 241 | +(post) and are shares of the references that fall inside the categorised corpus (Section~\ref{sec:m-citgraph}). | ||||
| 242 | +The LLM category is young, which depresses its share of cited works mechanically; the comparison with the | ||||
| 243 | +co-classification shares is therefore indicative, not a test.} Two flow changes deserve notice. The | ||||
| 244 | +self-citation share of computational complexity fell from | ||||
| 245 | +\ThyFlowComputationalComplexityToComputationalComplexityBaselinePct\% to | ||||
| 246 | +\ThyFlowComputationalComplexityToComputationalComplexityPostPct\% while its references to human--computer | ||||
| 247 | +interaction rose from \ThyFlowComputationalComplexityToHciBaselinePct\% to | ||||
| 248 | +\ThyFlowComputationalComplexityToHciPostPct\%; given the candidate list of | ||||
| 249 | +Section~\ref{sec:thy-landmarks}, this is more plausibly the category's contamination by social-science works | ||||
| 250 | +than a change in theoretical research, and it is recorded as a contested reading. Inbound citations to formal | ||||
| 251 | +verification from computer vision doubled, from \ThyInflowFormalVerificationFromComputerVisionBaselinePct\% to | ||||
| 252 | +\ThyInflowFormalVerificationFromComputerVisionPostPct\% of the category's inbound references, which is | ||||
| 253 | +consistent with the neural-network verification literature reviewed in Section~\ref{sec:thy-fv} being cited by | ||||
| 254 | +robustness work; no paper-level breakdown tests this. | ||||
| 255 | + | ||||
| 256 | +\section{Algorithms and data structures} | ||||
| 257 | +\label{sec:thy-algorithms} | ||||
| 258 | + | ||||
| 259 | +\subsection{What the window established} | ||||
| 260 | + | ||||
| 261 | +The category's largest results are not in the candidate list, and the first task of the review is to say what | ||||
| 262 | +they are, from verified records. Maximum flow and minimum-cost flow were brought to almost-linear time | ||||
| 263 | +\citep{chen2022maxflow}, closing a line of work that had run through interior-point methods and dynamic graph | ||||
| 264 | +data structures for a decade. Single-source shortest paths with negative weights were brought to near-linear | ||||
| 265 | +time by a combinatorial algorithm \citep{bernstein2022negative}, a result republished in the \emph{Journal of | ||||
| 266 | +the ACM} and \emph{Communications of the ACM} in 2025. The sorting barrier for directed single-source shortest | ||||
| 267 | +paths, the $\Omega(m + n \log n)$ cost that Dijkstra's algorithm with a priority queue cannot avoid, was | ||||
| 268 | +broken by a deterministic $O(m \log^{2/3} n)$ algorithm \citep{duan2025sorting}, the best paper of STOC~2025. | ||||
| 269 | +In data structures, the open-addressing bounds conjectured by Yao in 1985 were shown to be beatable: without | ||||
| 270 | +reordering, a greedy scheme achieves expected amortised probe complexity $O(\log^2 \delta^{-1})$ at load | ||||
| 271 | +$1 - \delta$, against the conjectured $\Theta(\delta^{-1})$ \citep{farachcolton2024openaddressing}. These | ||||
| 272 | +four results, from 2022 to 2025, are what a specialist would list first for the window; none has the citation | ||||
| 273 | +profile that the ranking rewards, because theoretical results are cited by a small community over long | ||||
| 274 | +horizons, and the 36-month window and the field normalisation are both calibrated on the behaviour of | ||||
| 275 | +applied categories. | ||||
| 276 | + | ||||
| 277 | +What the corpus does record is the programme of \emph{algorithms with predictions}, which the baseline period | ||||
| 278 | +established and the window continued. The candidate list carries it through a 2024 arXiv re-posting of the 2018 | ||||
| 279 | +conference paper on ski rental and non-clairvoyant scheduling with machine-learned predictions, whose | ||||
| 280 | +abstract describes algorithms that are ``oblivious to the performance of the predictor, improve with better | ||||
| 281 | +predictions, but do not degrade much if the predictions are poor''; the programme's canonical references are | ||||
| 282 | +the journal version of competitive caching with learned advice \citep{lykouris2021caching} and the overview in | ||||
| 283 | +\emph{Communications of the ACM} \citep{mitzenmacher2022predictions}. The formal object is a pair of | ||||
| 284 | +guarantees, consistency (the competitive ratio when the prediction is exact) and robustness (the ratio under | ||||
| 285 | +arbitrary error), with a trade-off between them; it is the point at which online algorithms and machine | ||||
| 286 | +learning meet without either giving up its guarantees, and it is the one place in the category where the | ||||
| 287 | +``learning'' vocabulary of the window names a theory rather than a tool. The fine-grained complexity programme | ||||
| 288 | +that organised algorithms research at the baseline (Section~\ref{sec:p2-theory}) is represented in the | ||||
| 289 | +bibliography by the ICM survey \citep{vassilevskawilliams2019finegrained} and by the parameterized-algorithms | ||||
| 290 | +monograph \citep{cygan2015parameterized}. | ||||
| 291 | + | ||||
| 292 | +\subsection{What the candidate list contains} | ||||
| 293 | + | ||||
| 294 | +Of the \ThyAuditAlgorithmsN{} algorithms candidates, \ThyAuditAlgorithmsInScopeResearch{} are research | ||||
| 295 | +contributions and none is misclassified; the rest are textbook chapters, re-deposited classics, theses and | ||||
| 296 | +metaheuristics. The two post-period research candidates are modest and both carry citation counts that belong | ||||
| 297 | +to other works: the WADS~2025 paper extends Kd-trees to Bregman divergences, shows that the triangle inequality | ||||
| 298 | +is not needed for correct pruning, and reports two orders of magnitude over linear search for | ||||
| 299 | +Kullback--Leibler nearest-neighbour queries in up to a hundred dimensions \citep{pham2025kdtree}; the SEA~2024 | ||||
| 300 | +paper replaces the highest-degree branching rule of branch-and-reduce for maximum independent set with a graph | ||||
| 301 | +neural network \citep{langedal2024mis}. Both abstracts describe engineering contributions with experimental | ||||
| 302 | +evaluation, which is what algorithm engineering is; neither is the kind of result that defines the window. | ||||
| 303 | + | ||||
| 304 | +The other research genre in the list is the metaphor-based metaheuristic: the Arithmetic Optimization Algorithm | ||||
| 305 | +\citep{abualigah2021aoa} and the Aquila Optimizer \citep{abualigah2021aquila} in 2021, and the artificial | ||||
| 306 | +lemming, glider snake, gray langur and starfish optimisers in 2025--2026, published in engineering and | ||||
| 307 | +AI-review journals and evaluated on CEC benchmark functions and constrained design problems. The abstracts share | ||||
| 308 | +a template, a natural metaphor mathematically modelled into exploration and exploitation phases and | ||||
| 309 | +``outperforming or matching'' a dozen earlier metaheuristics. The operational-research literature has argued | ||||
| 310 | +since \citet{sorensen2013metaphor} that the genre re-describes existing methods under new names and that its | ||||
| 311 | +benchmarks do not discriminate; the position was restated in a ``call for action'' \citep{aranha2021metaphor} | ||||
| 312 | +and substantiated for six widely cited algorithms, shown to be equivalent to or special cases of earlier | ||||
| 313 | +methods \citep{camachovillalon2022exposing}. The corpus does not adjudicate this dispute; what it shows is | ||||
| 314 | +the genre's weight in the highly cited stratum of the algorithms category, where six of twenty-five candidates | ||||
| 315 | +belong to it and where the post-2022 research articles with the highest normalised scores are of this kind. | ||||
| 316 | +Whether the genre grows faster than the rest of the category and absorbs a growing share of its citations is an | ||||
| 317 | +open problem of Part~X (G-P7), answerable with a keyword-defined subseries. | ||||
| 318 | + | ||||
| 319 | +\subsection{Algorithms discovered by learning systems} | ||||
| 320 | + | ||||
| 321 | +Three Nature papers and one preprint report algorithms found by reinforcement learning or program search: | ||||
| 322 | +faster matrix-multiplication tensor decompositions for small sizes \citep{fawzi2022alphatensor}, shorter | ||||
| 323 | +sorting networks for short sequences, merged into a standard library \citep{mankowitz2023alphadev}, new | ||||
| 324 | +constructions for the cap-set problem and bin packing from evolved programs \citep{romeraparedes2024funsearch}, | ||||
| 325 | +and, as a 2025 preprint, a coding agent that reports improvements on a collection of open mathematical and | ||||
| 326 | +algorithmic problems \citep{novikov2025alphaevolve}. All four are reports by the developers; the improvements | ||||
| 327 | +are on specific instances; and no independent reproduction or extension by the theory community appears in the | ||||
| 328 | +verified bibliography. The question these works pose to the category, whether search over programs changes how | ||||
| 329 | +algorithms are designed or only how constants are tuned, is one the corpus cannot answer at this granularity; | ||||
| 330 | +it is stated as open problem G-P9. The honest position is that the window's algorithmic breakthroughs were | ||||
| 331 | +obtained by people, and that the learned-discovery results are promising, contested in scope, and awaiting | ||||
| 332 | +replication. | ||||
| 333 | + | ||||
| 334 | +\section{Computational complexity} | ||||
| 335 | +\label{sec:thy-complexity} | ||||
| 336 | + | ||||
| 337 | +The category's statistics are the least trustworthy of the six: \ThyAuditComputationalComplexityMisclassified{} | ||||
| 338 | +of its \ThyAuditComputationalComplexityN{} candidates are outside computing, and the flow and co-occurrence | ||||
| 339 | +changes of Section~\ref{sec:thy-statistics} are to that extent artefacts. The two in-scope research candidates | ||||
| 340 | +are a STOC~2019 paper and a 2026 \emph{Theoretical Computer Science} article on SAT in spiking neural P systems | ||||
| 341 | +(no abstract). The STOC paper is a genuine landmark of the baseline: its abstract describes the reachability | ||||
| 342 | +problem for Petri nets as open since the 1960s, with Lipton's 1976 exponential-space lower bound unimproved, and | ||||
| 343 | +establishes a non-elementary lower bound \citep{czerwinski2019petri}. The story was completed within the | ||||
| 344 | +baseline period, when reachability in vector addition systems was shown Ackermann-complete | ||||
| 345 | +\citep{czerwinski2021ackermann,leroux2021notprimitive}, matching the upper bound of Leroux and Schmitz; the | ||||
| 346 | +window inherits a closed problem whose closure the corpus ranks below a technology-acceptance survey. | ||||
| 347 | + | ||||
| 348 | +The window's own contribution to complexity theory that a specialist would name first is the simulation of | ||||
| 349 | +time $t$ in space $O(\sqrt{t \log t})$ \citep{williams2025sqrtspace}, which improves the Hopcroft--Paul--Valiant | ||||
| 350 | +simulation of 1975 after fifty years and implies, through a known argument, that there are problems solvable | ||||
| 351 | +in linear space that need nearly quadratic time on multitape machines; the paper is a STOC~2025 result | ||||
| 352 | +republished in the \emph{Journal of the ACM} in 2026. In additive combinatorics, the strong bounds for | ||||
| 353 | +three-term arithmetic progressions \citep{kelley2023threeap} brought the Roth-type exponent to a form conjectured | ||||
| 354 | +since Behrend. Neither appears in the candidate list; both enter through verified records, and the general | ||||
| 355 | +point is the one made for algorithms: results of this kind are cited slowly and by few, and a 36-month | ||||
| 356 | +field-normalised window does not see them. | ||||
| 357 | + | ||||
| 358 | +The boundary with language models is where the category's co-classification share grew, yet the relevant | ||||
| 359 | +works are mostly classified elsewhere. The proof that transformers with logarithmic precision lie in uniform | ||||
| 360 | +$\mathrm{TC}^0$ \citep{merrill2023parallelism}, and the survey of what formal languages transformers can | ||||
| 361 | +express \citep{strobl2024survey}, bound what a single forward pass can compute and make the case that | ||||
| 362 | +chain-of-thought computation is where additional expressive power must come from; they are, in the verified | ||||
| 363 | +bibliography, the window's main contribution of complexity theory to the study of the models that dominate | ||||
| 364 | +Part~V. Whether such works are classified under computational complexity or under natural language processing | ||||
| 365 | +is an empirical question about the taxonomy (open problem G-P8). The remainder of the candidate list is | ||||
| 366 | +teaching material, including the fourth edition of the Kolmogorov-complexity monograph \citep{li2019kolmogorov} | ||||
| 367 | +and undergraduate chapters on complexity and on parameterized complexity, and re-deposited classics, including | ||||
| 368 | +Turing's 1936 paper and Wolfram's cellular-automata chapters. | ||||
| 369 | + | ||||
| 370 | +\section{Programming languages} | ||||
| 371 | +\label{sec:thy-pl} | ||||
| 372 | + | ||||
| 373 | +The baseline lineage was set out in Section~\ref{sec:p2-pl}: Rust and its semantic foundation | ||||
| 374 | +\citep{matsakis2014rust,jung2017rustbelt,jung2021saferust}, Lean~4 as a prover that is also a programming | ||||
| 375 | +language \citep{demoura2021lean4} with its mathematical library \citep{mathlib2020}, and the verified | ||||
| 376 | +artefacts CompCert \citep{leroy2009compcert} and seL4 \citep{klein2009sel4} that showed production software | ||||
| 377 | +can be proved correct. The candidate list records this lineage only obliquely, through three theses of the | ||||
| 378 | +L4/seL4 school re-deposited in 2022 (separation logic for programs in virtual memory, formal memory models for | ||||
| 379 | +C systems code, a verified multiprocessor kernel design), which the ranking places in the post period. In the | ||||
| 380 | +window the lineage continued with Verus \citep{lattuada2024verus}, a verification framework for Rust programs | ||||
| 381 | +presented at SOSP~2024, which treats the ownership discipline of the language as the foundation for | ||||
| 382 | +specification and proof rather than as an obstacle to them; the content of Verus is known here from its verified | ||||
| 383 | +title and venue only. | ||||
| 384 | + | ||||
| 385 | +Of the \ThyAuditProgrammingLanguagesN{} candidates, \ThyAuditProgrammingLanguagesInScopeResearch{} are | ||||
| 386 | +research contributions. Two of them mark the entry of language models into the community's top venue in | ||||
| 387 | +2024. LLift \citep{li2024llift} combines static analysis with an LLM to find use-before-initialisation bugs in | ||||
| 388 | +the Linux kernel; its abstract reports post-constraint guidance of the path analysis, procedures for handling | ||||
| 389 | +``the unpredictable nature of LLMs'', and four previously unknown bugs acknowledged by the kernel community. | ||||
| 390 | +WhiteFox \citep{yang2024whitefox} is described by its abstract as the first white-box compiler fuzzer that uses | ||||
| 391 | +an LLM reading the source code of optimisations to generate test programs that exercise them, against | ||||
| 392 | +black- and grey-box fuzzing that lacks that information and symbolic execution that does not scale to a | ||||
| 393 | +compiler. Both papers report bugs found; neither reports a guarantee, and the abstracts do not claim one. The | ||||
| 394 | +natural comparison is with the two instruments that the compiler community built for correctness before the | ||||
| 395 | +window, Csmith's random differential testing, which found hundreds of bugs in production C compilers | ||||
| 396 | +\citep{yang2011csmith}, and Alive2's bounded translation validation of LLVM optimisations | ||||
| 397 | +\citep{lopes2021alive2}; whether LLM-generated tests can be combined with validation of that kind, so that the | ||||
| 398 | +share of reported bugs that are confirmed can be measured per optimisation pass, is open problem G-P4. The | ||||
| 399 | +third research candidate, ZEUS \citep{kalra2018zeus}, is the baseline representative of program analysis for a | ||||
| 400 | +new target: abstract interpretation and constrained Horn clauses applied to smart contracts, with an abstract | ||||
| 401 | +reporting that about 95\% of 22,400 contracts analysed were vulnerable. The universal fuzzer of | ||||
| 402 | +\citet{xia2024fuzz4all} and the LLM-based bug-reproduction study of \citet{kang2023fewshottesters} are the | ||||
| 403 | +software-engineering counterparts of the same move. | ||||
| 404 | + | ||||
| 405 | +The category's vocabulary moved more than any except software engineering's: co-classification with the LLM | ||||
| 406 | +category reached \ThyCoocProgrammingLanguagesLlmTwoZeroTwoFivePct\% in 2025. Its references to LLM works are | ||||
| 407 | +\ThyFlowProgrammingLanguagesToLlmPostPct\%, and its structural ties are unchanged, with | ||||
| 408 | +\ThyFlowProgrammingLanguagesToFormalVerificationPostPct\% of its references going to formal verification and | ||||
| 409 | +\ThyFlowProgrammingLanguagesToSoftwareEngineeringPostPct\% to software engineering in the post period, against | ||||
| 410 | +\ThyFlowProgrammingLanguagesToFormalVerificationBaselinePct\% and | ||||
| 411 | +\ThyFlowProgrammingLanguagesToSoftwareEngineeringBaselinePct\% at the baseline. Programming languages is, in | ||||
| 412 | +the citation graph, the hinge between verification and engineering, and the window did not move the hinge. | ||||
| 413 | + | ||||
| 414 | +\section{Compilers} | ||||
| 415 | +\label{sec:thy-compilers} | ||||
| 416 | + | ||||
| 417 | +Compilers is the smallest of the six categories, \ThySharePrimaryCompilersTwoZeroTwoFivePct\% of primary labels | ||||
| 418 | +in 2025, and the most lexically contaminated after complexity. Its in-scope candidates nevertheless trace a | ||||
| 419 | +coherent line. At the baseline, MLIR \citep{lattner2021mlir} is the infrastructure whose abstract promises to | ||||
| 420 | +reduce ``the cost of building domain specific compilers'' and to connect existing ones; Tiramisu | ||||
| 421 | +\citep{baghdadi2019tiramisu} is a polyhedral compiler whose scheduling language separates algorithm, loop | ||||
| 422 | +transformation, data layout and communication so that one algorithm can target multicores, GPUs and | ||||
| 423 | +distributed machines; ExaStencils is a four-layer domain-specific language for multigrid solvers. Two surveys | ||||
| 424 | +establish that learned compiler heuristics are a programme older than the window: the \emph{Proceedings of the | ||||
| 425 | +IEEE} survey of machine learning in compiler optimisation \citep{wang2018mlcompiler} and the \emph{Computing | ||||
| 426 | +Surveys} review of autotuning, optimisation selection and phase ordering \citep{ashouri2018autotuning}, which | ||||
| 427 | +dates the first attempts to the mid-1990s. Three MIT and Delaware theses on learned optimisation selection and | ||||
| 428 | +on sparse tensor-algebra compilation complete the baseline picture. | ||||
| 429 | + | ||||
| 430 | +The post-period candidates show the programme's change of scale. LLM4VV \citep{munley2024llm4vv} uses language | ||||
| 431 | +models to generate a validation test suite for directive-based parallel programming compilers (metadata only; | ||||
| 432 | +no abstract). LLM Compiler \citep{cummins2025llmcompiler} is a suite of foundation models built on Code Llama | ||||
| 433 | +and trained, per its abstract, on 546 billion tokens of LLVM intermediate representation and assembly, then | ||||
| 434 | +instruction-tuned to predict and emulate compiler behaviour; fine-tuned versions reach 77\% of the optimising | ||||
| 435 | +potential of an autotuning search for code size and disassemble x86-64 and ARM assembly back to LLVM-IR. The | ||||
| 436 | +preprint that preceded it \citep{cummins2023llmcompiler} framed the task as predicting optimisation-pass | ||||
| 437 | +sequences. The shift is from learning a heuristic inside a compiler, with features chosen by compiler writers, | ||||
| 438 | +to pre-training a model on the compiler's own artefacts; what the shift leaves open is the question that | ||||
| 439 | +compiler research has always had to answer last, correctness, which the field handled through differential | ||||
| 440 | +testing \citep{yang2011csmith} and translation validation \citep{lopes2021alive2} rather than through model | ||||
| 441 | +evaluation. Co-classification with the LLM category reached \ThyCoocCompilersLlmTwoZeroTwoFivePct\% in 2025; | ||||
| 442 | +references to LLM works are \ThyFlowCompilersToLlmPostPct\%, and the category's citations continue to go | ||||
| 443 | +first to computer architecture (\ThyFlowCompilersToComputerArchitecturePostPct\% of references in the post | ||||
| 444 | +period) and to itself. The deep-learning compiler stack recorded in the main bibliography, TVM, Triton and | ||||
| 445 | +PyTorch~2 \citep{chen2018tvm,tillet2019triton,ansel2024pytorch2}, and the image-processing DSL that preceded | ||||
| 446 | +it \citep{ragankelley2013halide}, belong to Part~VI and are only noted here as the other post-2018 development | ||||
| 447 | +of the category. | ||||
| 448 | + | ||||
| 449 | +\section{Formal verification} | ||||
| 450 | +\label{sec:thy-fv} | ||||
| 451 | + | ||||
| 452 | +Formal verification has the most in-scope research candidates of the six | ||||
| 453 | +(\ThyAuditFormalVerificationInScopeResearch{} of \ThyAuditFormalVerificationN{}) and the clearest internal | ||||
| 454 | +structure, which the review follows: solvers and competitions, neural-network verification, verification of new | ||||
| 455 | +targets, learned theorem proving, and the theory of verification. | ||||
| 456 | + | ||||
| 457 | +\paragraph{Solvers and competitions.} | ||||
| 458 | +cvc5 \citep{barbosa2022cvc5} is the system description of the SMT solver that succeeded CVC4, evaluated on all | ||||
| 459 | +SMT-LIB benchmarks against CVC4 and Z3; the Handbook of Satisfiability in its second edition | ||||
| 460 | +\citep{biere2021handbook} is the reference work, present twice in the candidate list under two records. The | ||||
| 461 | +SV-COMP~2024 report \citep{beyer2024svcomp} describes the largest edition of the software-verification | ||||
| 462 | +competition to that date, 76 tools from 34 teams on 30,300 verification tasks for C programs, with a second | ||||
| 463 | +track for witness validation and a new witness format. Competitions of this kind are the category's benchmark | ||||
| 464 | +culture, and they are a point of contrast with the benchmark culture of Part~V: the tasks are fixed programs | ||||
| 465 | +with fixed specifications, a tool's result is a proof or a counterexample that a validator checks, and the | ||||
| 466 | +score is not self-reported. | ||||
| 467 | + | ||||
| 468 | +\paragraph{Neural-network verification.} | ||||
| 469 | +The baseline built a sub-field: Reluplex \citep{katz2017reluplex} extended the simplex method to ReLU | ||||
| 470 | +constraints; the 2018 symbolic-interval preprint in the candidate list argued, per its abstract, that existing | ||||
| 471 | +analyses ``struggle to scale beyond very small networks'' and proposed interval propagation that does; | ||||
| 472 | +DeepPoly \citep{singh2019deeppoly} introduced an abstract domain of floating-point polyhedra with intervals and | ||||
| 473 | +transformers for ReLU, sigmoid, tanh and maxpool, and reported the first proofs of robustness to rotations | ||||
| 474 | +with interpolation; Marabou \citep{katz2019marabou} packaged SMT-based verification as a framework accepting | ||||
| 475 | +TensorFlow models. The VNN-COMP report \citep{brix2023vnncomp} summarises the first three years of the | ||||
| 476 | +competition that organised the sub-field. The doubling of formal verification's inbound citations from | ||||
| 477 | +computer vision in the post period (Section~\ref{sec:thy-statistics}) is consistent with this literature being | ||||
| 478 | +cited by robustness work, and the sub-field is the main channel through which the category meets the | ||||
| 479 | +machine-learning categories; whether it has guarantees that survive the scale of the models of Part~V is not a | ||||
| 480 | +question these abstracts address. | ||||
| 481 | + | ||||
| 482 | +\paragraph{New targets.} | ||||
| 483 | +VerX \citep{permenev2020verx} is described by its abstract as the first automated verifier of functional | ||||
| 484 | +properties of Ethereum smart contracts, combining reduction of temporal properties to reachability, a symbolic | ||||
| 485 | +execution engine for the Ethereum virtual machine, and delayed predicate abstraction, evaluated on 83 | ||||
| 486 | +properties of 12 projects. The answer-set-programming monitor for automated driving \citep{klampfl2024asp} | ||||
| 487 | +and the theses on parametric and multi-objective Markov models represent runtime and probabilistic | ||||
| 488 | +verification; the Warwick thesis on model checking quantum protocols, re-deposited with a 2026 date, is a | ||||
| 489 | +baseline work of the previous decade. | ||||
| 490 | + | ||||
| 491 | +\paragraph{Learned theorem proving.} | ||||
| 492 | +The abstract attached in the corpus to the ITP~2023 paper, under a wrong title (Section~\ref{sec:thy-landmarks}), | ||||
| 493 | +is the clearest statement in the candidate set of what machine learning has done for automated reasoning: | ||||
| 494 | +MizAR \citep{jakubuv2023mizar} proves about 60\% of the theorems of the Mizar Mathematical Library in the hammer | ||||
| 495 | +setting and 75\% when given the premises of the human proofs, through the E and Vampire provers with the ENIGMA | ||||
| 496 | +and Deepire learned guidance and an incremental loop that grows a corpus of millions of proofs while training | ||||
| 497 | +stronger systems on it. This is the saturation-prover line. The language-model line is represented by Baldur | ||||
| 498 | +\citep{first2023baldur}, which generates and repairs whole Isabelle proofs; by LeanDojo | ||||
| 499 | +\citep{yang2023leandojo} and DeepSeek-Prover \citep{xin2024deepseekprover}, both preprints, which report | ||||
| 500 | +retrieval-augmented and synthetic-data-trained provers on Lean benchmarks; and by the position paper on formal | ||||
| 501 | +mathematical reasoning as a frontier \citep{yang2024formalmath}, also a preprint. The figures these works | ||||
| 502 | +report are on different benchmarks, under different compute budgets, with the possibility of overlap between | ||||
| 503 | +training corpora and test theorems that the abstracts do not exclude; a benchmark of theorems posted after the | ||||
| 504 | +training cut-off, with proof checking rather than self-reported success, is the direction stated as open | ||||
| 505 | +problem G-P5. The machine-learned mathematics of Part~V, olympiad geometry \citep{trinh2024alphageometry} and | ||||
| 506 | +program search \citep{romeraparedes2024funsearch}, is adjacent but distinct: it produces objects that a human or | ||||
| 507 | +a checker then verifies, where the works above produce the proofs. | ||||
| 508 | + | ||||
| 509 | +\paragraph{Theory of verification.} | ||||
| 510 | +Equality saturation as implemented in \emph{egg} \citep{willsey2021egg}, whose abstract describes e-graph | ||||
| 511 | +rewriting that ``only adds information'' and extraction by a user-supplied cost function, is a baseline result | ||||
| 512 | +republished as a research highlight in 2026; the quantitative interface automata of the CSL~2026 paper on reward | ||||
| 513 | +interfaces, whose abstract introduces multi-valued correctness requirements and best-effort satisfaction into | ||||
| 514 | +interface theories, is the one post-period research candidate of this kind. | ||||
| 515 | + | ||||
| 516 | +The category's primary share halved between 2018 and 2025 while its multi-label count grew by eleven per cent | ||||
| 517 | +(Table~\ref{tab:theory_counts}). The arithmetic of the argmax assignment (Section~\ref{sec:m-score}) suggests | ||||
| 518 | +why: a work that carries a verification label and an AI or security label loses the primary label to the | ||||
| 519 | +larger category when the scores are close, so the primary series understates a category whose work is | ||||
| 520 | +increasingly done in combination. Measuring the share of works whose primary label switched, by co-assigned | ||||
| 521 | +category and year, is open problem G-P10. | ||||
| 522 | + | ||||
| 523 | +\section{Software engineering} | ||||
| 524 | +\label{sec:thy-se} | ||||
| 525 | + | ||||
| 526 | +Software engineering is where the window's change is largest, best documented, and most contested. Its | ||||
| 527 | +baseline was not AI-free (Section~\ref{sec:p2-pl}): the Codex models \citep{chen2021codex}, the | ||||
| 528 | +code-generation benchmarks \citep{austin2021mbpp}, the systematic evaluation of code models | ||||
| 529 | +\citep{xu2022codelms} and the first productivity assessment of neural code completion | ||||
| 530 | +\citep{ziegler2022copilot} precede the reference date, and the survey of machine-learning testing | ||||
| 531 | +\citep{zhang2022mltesting}, 144 papers on testing properties, components and workflows of learned systems, | ||||
| 532 | +shows the category already organised around learned software as an object of test. The window's surveys | ||||
| 533 | +reverse the direction of the relation. The survey of software testing with large language models | ||||
| 534 | +\citep{wang2024llmtesting} analyses 102 studies and reports, per its abstract, that test-case preparation and | ||||
| 535 | +program repair are the most common uses; the two systematic reviews of language models for software | ||||
| 536 | +engineering \citep{fan2023llm4se,hou2024llm4se} and the code-generation survey \citep{jiang2024codegensurvey} | ||||
| 537 | +map the same territory. Method papers from the main bibliography and the Part~VII-a bibliography give the | ||||
| 538 | +texture: program repair in the era of pre-trained models \citep{xia2023apr}, bug reproduction from reports | ||||
| 539 | +\citep{kang2023fewshottesters}, universal fuzzing \citep{xia2024fuzz4all}, and the agentic setting defined by | ||||
| 540 | +SWE-bench \citep{jimenez2024swebench} and SWE-agent \citep{yang2024sweagent}, both preprints, in which a model | ||||
| 541 | +is asked to resolve a GitHub issue end to end. Among the candidates, APISS \citep{chen2025apiss} retrieves | ||||
| 542 | +functionality-equivalent APIs through LLM embeddings to migrate proof-of-concept exploits, and a 2026 paper | ||||
| 543 | +proposes a multi-scale temporal model for just-in-time defect prediction; both are method contributions judged | ||||
| 544 | +from their abstracts. | ||||
| 545 | + | ||||
| 546 | +\paragraph{Productivity: two findings that need reconciling.} | ||||
| 547 | +The most cited post-period empirical candidate reports three randomised field experiments at Microsoft, | ||||
| 548 | +Accenture and an unnamed Fortune~100 company, run by the firms in the course of business, in which a random | ||||
| 549 | +subset of 4,867 developers received an AI coding assistant; the pooled estimate is a 26.08\% increase in | ||||
| 550 | +completed tasks with a standard error of 10.3\%, each experiment being ``noisy'' on its own, and less | ||||
| 551 | +experienced developers adopting more and gaining more \citep{cui2026genaiwork}. The earlier controlled | ||||
| 552 | +experiment on a single task reported a larger speed-up \citep{peng2023copilot}, and the field studies of | ||||
| 553 | +generative AI at work outside software \citep{brynjolfsson2023genaiwork,noy2023productivity,dellacqua2023jagged} | ||||
| 554 | +report the same pattern of larger gains for less experienced workers. Against these stands a 2025 randomised | ||||
| 555 | +trial preprint in which experienced open-source developers working on repositories they maintain took longer | ||||
| 556 | +to complete issues with AI tools than without \citep{becker2025metr}. The two findings are not contradictory | ||||
| 557 | +once population and outcome are distinguished: enterprise developers completing assigned tasks versus expert | ||||
| 558 | +maintainers resolving issues in mature codebases they know well, and tasks completed versus time per issue. | ||||
| 559 | +But the reconciliation is a reading of two abstracts, the second of a preprint, and the design that would | ||||
| 560 | +settle it, a preregistered multi-site trial stratified by experience and repository maturity with time-to-merge | ||||
| 561 | +and defect rate as joint outcomes, has not been run (open problem G-P3). The user studies of how programmers | ||||
| 562 | +interact with code-generating models \citep{barke2023groundedcopilot,vaithilingam2022expectation,liang2024usability} | ||||
| 563 | +supply the mechanism, acceleration mode against exploration mode, that any such trial would need to measure. | ||||
| 564 | + | ||||
| 565 | +\paragraph{Security of generated code.} | ||||
| 566 | +The security evaluations agree with one another. Copilot's suggestions were found vulnerable at a substantial | ||||
| 567 | +rate in 2022 \citep{pearce2022asleep}; the CCS~2023 user study found that participants with an AI assistant | ||||
| 568 | +wrote less secure code and believed it more secure \citep{perry2023insecure}; and the ICPEC~2025 study in the | ||||
| 569 | +candidate list, evaluating three open models on a filtered Big-Vul subset with eight CWE categories, reports | ||||
| 570 | +``a sharp contrast between high detection rates and markedly poor classification accuracy'' | ||||
| 571 | +\citep{gasparlopes2025vuln}. High detection or completion paired with poor fine-grained security reasoning is | ||||
| 572 | +the consistent pattern across three designs, and it is the category's clearest negative result about the | ||||
| 573 | +models. | ||||
| 574 | + | ||||
| 575 | +\paragraph{The statistics.} | ||||
| 576 | +Software engineering has the highest co-classification with the LLM category of the six | ||||
| 577 | +(\ThyCoocSoftwareEngineeringLlmTwoZeroTwoFivePct\% of its works in 2025, from | ||||
| 578 | +\ThyCoocSoftwareEngineeringLlmTwoZeroTwoTwoPct\% in 2022), the largest post-period growth | ||||
| 579 | +(\ThyCagrMultilabelSoftwareEngineeringPostPct\% per year), the only positive level change at the reference | ||||
| 580 | +date and the only significant positive level change at the GPT-4 date (Section~\ref{sec:thy-statistics}). | ||||
| 581 | +Its references to LLM works are \ThyFlowSoftwareEngineeringToLlmPostPct\% in the post period, from | ||||
| 582 | +\ThyFlowSoftwareEngineeringToLlmBaselinePct\%, and inbound citations from the LLM category are | ||||
| 583 | +\ThyInflowSoftwareEngineeringFromLlmPostPct\% of its inbound references. Whether a vocabulary shift of this | ||||
| 584 | +size is a method shift, that is, whether the works co-classified with LLM cite LLM method papers at a higher | ||||
| 585 | +rate than the category's baseline, can be tested on the knowledge graph of Part~III once method nodes are | ||||
| 586 | +extracted for the high-priority set; it is open problem G-P2. The research-software case of the Astropy | ||||
| 587 | +project \citep[in the candidate list;][]{collberg2016repeatability} and the reproducibility literature of | ||||
| 588 | +Part~IX are the other side of the same category, and the only part of it where the window changed nothing | ||||
| 589 | +visible. | ||||
| 590 | + | ||||
| 591 | +\section{Synthesis} | ||||
| 592 | +\label{sec:thy-synthesis} | ||||
| 593 | + | ||||
| 594 | +Three conclusions follow, in decreasing order of evidential strength. | ||||
| 595 | + | ||||
| 596 | +The first is about the instrument. In the theory and software areas the automatically selected landmark | ||||
| 597 | +candidates do not identify landmarks: \ThyAuditVerdictInScopeResearch{} of \ThyAuditN{} are in-scope research | ||||
| 598 | +works, \ThyAuditIdCollision{} carry citation windows that belong to other works, and the rest are chapters with | ||||
| 599 | +generic titles, re-deposited classics, theses in small normalisation cells and lexical misclassifications. The | ||||
| 600 | +defects are identifiable and partly correctable at the selection stage, and the audit table | ||||
| 601 | +(Table~\ref{tab:theory_landmark_audit}) should accompany any use of the list; the major theoretical results of | ||||
| 602 | +the window, almost-linear flow, near-linear negative-weight shortest paths, the directed sorting barrier, | ||||
| 603 | +optimal open addressing, square-root-space simulation and the 3-progression bounds, are invisible to a | ||||
| 604 | +36-month field-normalised ranking and were brought into this review through verified records. The same | ||||
| 605 | +blindness will affect any bibliometric answer to RQ3 and RQ7 for these categories. | ||||
| 606 | + | ||||
| 607 | +The second is about the statistics. The six categories grew in absolute terms and declined in relative terms | ||||
| 608 | +after 2022; their slope changes at the reference date are positive but not distinguishable from pseudo-events | ||||
| 609 | +in the baseline; their vocabulary moved toward language models in proportion to their distance from | ||||
| 610 | +mathematics; and their citation behaviour moved little. For RQ2 the answer is that none of the six is among | ||||
| 611 | +the categories with structural change that survives the placebo test, and for RQ5 that the relationship with | ||||
| 612 | +the foundation-model categories changed in what these works say far more than in what they cite. | ||||
| 613 | + | ||||
| 614 | +The third is about the research. The substantive developments of the window are of two kinds that the corpus | ||||
| 615 | +treats very differently. Theoretical results were obtained by people, at the rate and in the venues of the | ||||
| 616 | +previous decade, and are cited accordingly. The entry of language models into verification, program analysis, | ||||
| 617 | +compiler testing and software engineering is visible in the co-classification statistics and in the abstracts | ||||
| 618 | +of the in-scope candidates, and it is where the open problems lie: guarantees for learned program analysis and | ||||
| 619 | +compiler testing (G-P4), comparability of learned provers (G-P5), the productivity effect on experienced | ||||
| 620 | +developers (G-P3), and the distinction between a vocabulary and a method (G-P2). What this half of Part~VII | ||||
| 621 | +can say with confidence is which of these questions the corpus can answer with its present instruments, and | ||||
| 622 | +which it cannot. | ||||
added
results/graph/semantic_crossing_yearly.csv
+361 −0
@@ -0,0 +1,361 @@ | |||||
| 1 | +year,primary_category,n_edges,share_cross_category | ||||
| 2 | +2018,ai_accelerators,6157,0.5181 | ||||
| 3 | +2018,ai_agents,30233,0.548 | ||||
| 4 | +2018,ai_safety,5267,0.569 | ||||
| 5 | +2018,algorithms,170250,0.3159 | ||||
| 6 | +2018,applied_cryptography,60756,0.4146 | ||||
| 7 | +2018,autonomous_scientific_discovery,495,0.9253 | ||||
| 8 | +2018,blockchain,35962,0.2207 | ||||
| 9 | +2018,cloud_computing,58539,0.4688 | ||||
| 10 | +2018,compilers,4635,0.8194 | ||||
| 11 | +2018,computational_biology,92838,0.1982 | ||||
| 12 | +2018,computational_complexity,20738,0.6304 | ||||
| 13 | +2018,computer_architecture,140938,0.3222 | ||||
| 14 | +2018,computer_graphics,63877,0.4237 | ||||
| 15 | +2018,computer_networks,446365,0.1723 | ||||
| 16 | +2018,computer_vision,463491,0.1823 | ||||
| 17 | +2018,cybersecurity,183347,0.3546 | ||||
| 18 | +2018,database_systems,102960,0.4521 | ||||
| 19 | +2018,deep_learning_architectures,74595,0.4948 | ||||
| 20 | +2018,distributed_computing,75532,0.5767 | ||||
| 21 | +2018,edge_computing,34523,0.5445 | ||||
| 22 | +2018,federated_learning,726,0.8209 | ||||
| 23 | +2018,formal_verification,33962,0.465 | ||||
| 24 | +2018,foundation_models,245,0.9061 | ||||
| 25 | +2018,generative_ai,5206,0.6498 | ||||
| 26 | +2018,hci,216031,0.2699 | ||||
| 27 | +2018,information_retrieval,71592,0.3751 | ||||
| 28 | +2018,llm,4817,0.8337 | ||||
| 29 | +2018,ml_theory,91278,0.4192 | ||||
| 30 | +2018,multimodal_ai,2851,0.7555 | ||||
| 31 | +2018,nlp,158887,0.2678 | ||||
| 32 | +2018,operating_systems,9487,0.8001 | ||||
| 33 | +2018,privacy_preserving_computation,19705,0.4446 | ||||
| 34 | +2018,programming_languages,27407,0.6148 | ||||
| 35 | +2018,quantum_computing,39415,0.1159 | ||||
| 36 | +2018,reinforcement_learning,22774,0.351 | ||||
| 37 | +2018,robotics,134764,0.242 | ||||
| 38 | +2018,scientific_computing,86496,0.4539 | ||||
| 39 | +2018,software_engineering,179454,0.2979 | ||||
| 40 | +2018,speech_audio,77812,0.1873 | ||||
| 41 | +2018,sustainable_computing,3256,0.8068 | ||||
| 42 | +2019,ai_accelerators,7263,0.531 | ||||
| 43 | +2019,ai_agents,30144,0.5601 | ||||
| 44 | +2019,ai_safety,8150,0.5583 | ||||
| 45 | +2019,algorithms,179415,0.3109 | ||||
| 46 | +2019,applied_cryptography,68116,0.4176 | ||||
| 47 | +2019,autonomous_scientific_discovery,715,0.9105 | ||||
| 48 | +2019,blockchain,52593,0.2132 | ||||
| 49 | +2019,cloud_computing,57803,0.4702 | ||||
| 50 | +2019,compilers,4164,0.8343 | ||||
| 51 | +2019,computational_biology,109708,0.2013 | ||||
| 52 | +2019,computational_complexity,22012,0.6335 | ||||
| 53 | +2019,computer_architecture,137403,0.3238 | ||||
| 54 | +2019,computer_graphics,60635,0.4342 | ||||
| 55 | +2019,computer_networks,447623,0.1791 | ||||
| 56 | +2019,computer_vision,554061,0.1766 | ||||
| 57 | +2019,cybersecurity,202702,0.3475 | ||||
| 58 | +2019,database_systems,107527,0.4559 | ||||
| 59 | +2019,deep_learning_architectures,97754,0.4835 | ||||
| 60 | +2019,distributed_computing,76991,0.5673 | ||||
| 61 | +2019,edge_computing,45246,0.5424 | ||||
| 62 | +2019,federated_learning,1161,0.8243 | ||||
| 63 | +2019,formal_verification,29780,0.4782 | ||||
| 64 | +2019,foundation_models,506,0.8972 | ||||
| 65 | +2019,generative_ai,8293,0.6549 | ||||
| 66 | +2019,hci,232616,0.266 | ||||
| 67 | +2019,information_retrieval,74631,0.3586 | ||||
| 68 | +2019,llm,6560,0.8125 | ||||
| 69 | +2019,ml_theory,118641,0.4323 | ||||
| 70 | +2019,multimodal_ai,4264,0.7207 | ||||
| 71 | +2019,nlp,179279,0.2567 | ||||
| 72 | +2019,operating_systems,10082,0.7343 | ||||
| 73 | +2019,privacy_preserving_computation,21380,0.4506 | ||||
| 74 | +2019,programming_languages,28291,0.6236 | ||||
| 75 | +2019,quantum_computing,48912,0.1143 | ||||
| 76 | +2019,reinforcement_learning,34163,0.3233 | ||||
| 77 | +2019,robotics,151045,0.2428 | ||||
| 78 | +2019,scientific_computing,96671,0.4379 | ||||
| 79 | +2019,software_engineering,181111,0.3012 | ||||
| 80 | +2019,speech_audio,85969,0.1821 | ||||
| 81 | +2019,sustainable_computing,3832,0.7985 | ||||
| 82 | +2020,ai_accelerators,8555,0.5239 | ||||
| 83 | +2020,ai_agents,29472,0.5535 | ||||
| 84 | +2020,ai_safety,12179,0.538 | ||||
| 85 | +2020,algorithms,178532,0.3103 | ||||
| 86 | +2020,applied_cryptography,66281,0.408 | ||||
| 87 | +2020,autonomous_scientific_discovery,1271,0.8607 | ||||
| 88 | +2020,blockchain,59231,0.2082 | ||||
| 89 | +2020,cloud_computing,52776,0.4858 | ||||
| 90 | +2020,compilers,4503,0.8368 | ||||
| 91 | +2020,computational_biology,135533,0.1698 | ||||
| 92 | +2020,computational_complexity,21487,0.6388 | ||||
| 93 | +2020,computer_architecture,127745,0.3222 | ||||
| 94 | +2020,computer_graphics,60112,0.4218 | ||||
| 95 | +2020,computer_networks,417940,0.1881 | ||||
| 96 | +2020,computer_vision,577782,0.1761 | ||||
| 97 | +2020,cybersecurity,210760,0.3413 | ||||
| 98 | +2020,database_systems,98900,0.4658 | ||||
| 99 | +2020,deep_learning_architectures,120600,0.4763 | ||||
| 100 | +2020,distributed_computing,72448,0.5751 | ||||
| 101 | +2020,edge_computing,47740,0.5417 | ||||
| 102 | +2020,federated_learning,1722,0.7706 | ||||
| 103 | +2020,formal_verification,29104,0.4773 | ||||
| 104 | +2020,foundation_models,822,0.9051 | ||||
| 105 | +2020,generative_ai,10311,0.6379 | ||||
| 106 | +2020,hci,229085,0.2661 | ||||
| 107 | +2020,information_retrieval,73086,0.3568 | ||||
| 108 | +2020,llm,11303,0.7639 | ||||
| 109 | +2020,ml_theory,136518,0.4254 | ||||
| 110 | +2020,multimodal_ai,5280,0.7193 | ||||
| 111 | +2020,nlp,199186,0.2716 | ||||
| 112 | +2020,operating_systems,8975,0.7552 | ||||
| 113 | +2020,privacy_preserving_computation,26886,0.4459 | ||||
| 114 | +2020,programming_languages,25835,0.6379 | ||||
| 115 | +2020,quantum_computing,54478,0.1196 | ||||
| 116 | +2020,reinforcement_learning,41970,0.3203 | ||||
| 117 | +2020,robotics,159762,0.2454 | ||||
| 118 | +2020,scientific_computing,105432,0.4301 | ||||
| 119 | +2020,software_engineering,178500,0.2989 | ||||
| 120 | +2020,speech_audio,86196,0.1777 | ||||
| 121 | +2020,sustainable_computing,3712,0.7702 | ||||
| 122 | +2021,ai_accelerators,10185,0.4986 | ||||
| 123 | +2021,ai_agents,29211,0.5379 | ||||
| 124 | +2021,ai_safety,15348,0.5426 | ||||
| 125 | +2021,algorithms,185099,0.3075 | ||||
| 126 | +2021,applied_cryptography,65691,0.4139 | ||||
| 127 | +2021,autonomous_scientific_discovery,1520,0.8138 | ||||
| 128 | +2021,blockchain,68145,0.2024 | ||||
| 129 | +2021,cloud_computing,53243,0.4808 | ||||
| 130 | +2021,compilers,4166,0.8346 | ||||
| 131 | +2021,computational_biology,145854,0.1767 | ||||
| 132 | +2021,computational_complexity,21138,0.6401 | ||||
| 133 | +2021,computer_architecture,126775,0.3227 | ||||
| 134 | +2021,computer_graphics,59871,0.432 | ||||
| 135 | +2021,computer_networks,410121,0.1944 | ||||
| 136 | +2021,computer_vision,681479,0.1697 | ||||
| 137 | +2021,cybersecurity,221529,0.3434 | ||||
| 138 | +2021,database_systems,98155,0.4755 | ||||
| 139 | +2021,deep_learning_architectures,148163,0.4828 | ||||
| 140 | +2021,distributed_computing,69103,0.5739 | ||||
| 141 | +2021,edge_computing,55656,0.5292 | ||||
| 142 | +2021,federated_learning,4389,0.7471 | ||||
| 143 | +2021,formal_verification,30190,0.4816 | ||||
| 144 | +2021,foundation_models,1619,0.9036 | ||||
| 145 | +2021,generative_ai,13848,0.6419 | ||||
| 146 | +2021,hci,230614,0.2602 | ||||
| 147 | +2021,information_retrieval,75040,0.3477 | ||||
| 148 | +2021,llm,16560,0.7556 | ||||
| 149 | +2021,ml_theory,145324,0.4186 | ||||
| 150 | +2021,multimodal_ai,6653,0.7089 | ||||
| 151 | +2021,nlp,220164,0.2669 | ||||
| 152 | +2021,operating_systems,8703,0.7807 | ||||
| 153 | +2021,privacy_preserving_computation,29902,0.4454 | ||||
| 154 | +2021,programming_languages,25656,0.6097 | ||||
| 155 | +2021,quantum_computing,64423,0.1201 | ||||
| 156 | +2021,reinforcement_learning,50682,0.3234 | ||||
| 157 | +2021,robotics,174798,0.2535 | ||||
| 158 | +2021,scientific_computing,112748,0.4206 | ||||
| 159 | +2021,software_engineering,169864,0.3095 | ||||
| 160 | +2021,speech_audio,93912,0.1818 | ||||
| 161 | +2021,sustainable_computing,3960,0.7631 | ||||
| 162 | +2022,ai_accelerators,11638,0.4852 | ||||
| 163 | +2022,ai_agents,27771,0.5497 | ||||
| 164 | +2022,ai_safety,19034,0.5405 | ||||
| 165 | +2022,algorithms,181230,0.311 | ||||
| 166 | +2022,applied_cryptography,67510,0.4226 | ||||
| 167 | +2022,autonomous_scientific_discovery,1523,0.8772 | ||||
| 168 | +2022,blockchain,80994,0.2108 | ||||
| 169 | +2022,cloud_computing,51091,0.4913 | ||||
| 170 | +2022,compilers,3954,0.8462 | ||||
| 171 | +2022,computational_biology,150600,0.1813 | ||||
| 172 | +2022,computational_complexity,19809,0.6552 | ||||
| 173 | +2022,computer_architecture,120398,0.3229 | ||||
| 174 | +2022,computer_graphics,58221,0.4419 | ||||
| 175 | +2022,computer_networks,401999,0.199 | ||||
| 176 | +2022,computer_vision,718729,0.1678 | ||||
| 177 | +2022,cybersecurity,229807,0.3474 | ||||
| 178 | +2022,database_systems,96781,0.4772 | ||||
| 179 | +2022,deep_learning_architectures,159287,0.4821 | ||||
| 180 | +2022,distributed_computing,65426,0.589 | ||||
| 181 | +2022,edge_computing,62183,0.5375 | ||||
| 182 | +2022,federated_learning,6353,0.7244 | ||||
| 183 | +2022,formal_verification,26817,0.4724 | ||||
| 184 | +2022,foundation_models,2359,0.8978 | ||||
| 185 | +2022,generative_ai,15112,0.6619 | ||||
| 186 | +2022,hci,246850,0.2561 | ||||
| 187 | +2022,information_retrieval,77390,0.3335 | ||||
| 188 | +2022,llm,21803,0.7023 | ||||
| 189 | +2022,ml_theory,147928,0.4129 | ||||
| 190 | +2022,multimodal_ai,9216,0.6969 | ||||
| 191 | +2022,nlp,224621,0.2853 | ||||
| 192 | +2022,operating_systems,7330,0.7935 | ||||
| 193 | +2022,privacy_preserving_computation,32588,0.4538 | ||||
| 194 | +2022,programming_languages,24406,0.6179 | ||||
| 195 | +2022,quantum_computing,68971,0.1223 | ||||
| 196 | +2022,reinforcement_learning,54682,0.3177 | ||||
| 197 | +2022,robotics,184921,0.2512 | ||||
| 198 | +2022,scientific_computing,113486,0.4277 | ||||
| 199 | +2022,software_engineering,171531,0.3128 | ||||
| 200 | +2022,speech_audio,97128,0.1778 | ||||
| 201 | +2022,sustainable_computing,4700,0.7532 | ||||
| 202 | +2023,ai_accelerators,13046,0.4962 | ||||
| 203 | +2023,ai_agents,31675,0.5545 | ||||
| 204 | +2023,ai_safety,26172,0.5404 | ||||
| 205 | +2023,algorithms,184947,0.3131 | ||||
| 206 | +2023,applied_cryptography,74594,0.4263 | ||||
| 207 | +2023,autonomous_scientific_discovery,2873,0.8173 | ||||
| 208 | +2023,blockchain,101857,0.2044 | ||||
| 209 | +2023,cloud_computing,56168,0.4943 | ||||
| 210 | +2023,compilers,4189,0.835 | ||||
| 211 | +2023,computational_biology,159302,0.1853 | ||||
| 212 | +2023,computational_complexity,20196,0.6614 | ||||
| 213 | +2023,computer_architecture,131865,0.3229 | ||||
| 214 | +2023,computer_graphics,67297,0.4532 | ||||
| 215 | +2023,computer_networks,433815,0.2031 | ||||
| 216 | +2023,computer_vision,861207,0.1686 | ||||
| 217 | +2023,cybersecurity,274554,0.335 | ||||
| 218 | +2023,database_systems,103963,0.4807 | ||||
| 219 | +2023,deep_learning_architectures,184269,0.4815 | ||||
| 220 | +2023,distributed_computing,69801,0.5846 | ||||
| 221 | +2023,edge_computing,68563,0.5329 | ||||
| 222 | +2023,federated_learning,9556,0.7345 | ||||
| 223 | +2023,formal_verification,30583,0.4822 | ||||
| 224 | +2023,foundation_models,4193,0.9094 | ||||
| 225 | +2023,generative_ai,25933,0.6454 | ||||
| 226 | +2023,hci,288325,0.2557 | ||||
| 227 | +2023,information_retrieval,87299,0.328 | ||||
| 228 | +2023,llm,77315,0.51 | ||||
| 229 | +2023,ml_theory,171365,0.419 | ||||
| 230 | +2023,multimodal_ai,14035,0.702 | ||||
| 231 | +2023,nlp,259107,0.3093 | ||||
| 232 | +2023,operating_systems,8169,0.7975 | ||||
| 233 | +2023,privacy_preserving_computation,38950,0.4841 | ||||
| 234 | +2023,programming_languages,25578,0.63 | ||||
| 235 | +2023,quantum_computing,82434,0.1331 | ||||
| 236 | +2023,reinforcement_learning,65006,0.3221 | ||||
| 237 | +2023,robotics,209839,0.2523 | ||||
| 238 | +2023,scientific_computing,120013,0.4333 | ||||
| 239 | +2023,software_engineering,191926,0.3164 | ||||
| 240 | +2023,speech_audio,111431,0.1851 | ||||
| 241 | +2023,sustainable_computing,6610,0.7452 | ||||
| 242 | +2024,ai_accelerators,14257,0.5063 | ||||
| 243 | +2024,ai_agents,37986,0.5616 | ||||
| 244 | +2024,ai_safety,41126,0.5402 | ||||
| 245 | +2024,algorithms,202214,0.3203 | ||||
| 246 | +2024,applied_cryptography,84262,0.4261 | ||||
| 247 | +2024,autonomous_scientific_discovery,3282,0.848 | ||||
| 248 | +2024,blockchain,111052,0.2038 | ||||
| 249 | +2024,cloud_computing,64842,0.49 | ||||
| 250 | +2024,compilers,4755,0.8307 | ||||
| 251 | +2024,computational_biology,179730,0.1943 | ||||
| 252 | +2024,computational_complexity,22786,0.6901 | ||||
| 253 | +2024,computer_architecture,141510,0.322 | ||||
| 254 | +2024,computer_graphics,75793,0.4667 | ||||
| 255 | +2024,computer_networks,455866,0.2086 | ||||
| 256 | +2024,computer_vision,990283,0.1717 | ||||
| 257 | +2024,cybersecurity,344043,0.3253 | ||||
| 258 | +2024,database_systems,108454,0.4992 | ||||
| 259 | +2024,deep_learning_architectures,214093,0.4833 | ||||
| 260 | +2024,distributed_computing,75588,0.5921 | ||||
| 261 | +2024,edge_computing,77537,0.5292 | ||||
| 262 | +2024,federated_learning,12221,0.7153 | ||||
| 263 | +2024,formal_verification,29732,0.4824 | ||||
| 264 | +2024,foundation_models,7719,0.8851 | ||||
| 265 | +2024,generative_ai,49797,0.6391 | ||||
| 266 | +2024,hci,351074,0.2598 | ||||
| 267 | +2024,information_retrieval,99364,0.3423 | ||||
| 268 | +2024,llm,191956,0.4647 | ||||
| 269 | +2024,ml_theory,200633,0.4223 | ||||
| 270 | +2024,multimodal_ai,25075,0.72 | ||||
| 271 | +2024,nlp,296010,0.3364 | ||||
| 272 | +2024,operating_systems,8522,0.7941 | ||||
| 273 | +2024,privacy_preserving_computation,51216,0.4972 | ||||
| 274 | +2024,programming_languages,25672,0.648 | ||||
| 275 | +2024,quantum_computing,99990,0.1384 | ||||
| 276 | +2024,reinforcement_learning,77161,0.3317 | ||||
| 277 | +2024,robotics,248034,0.261 | ||||
| 278 | +2024,scientific_computing,132038,0.4453 | ||||
| 279 | +2024,software_engineering,222332,0.3241 | ||||
| 280 | +2024,speech_audio,123868,0.1872 | ||||
| 281 | +2024,sustainable_computing,7770,0.739 | ||||
| 282 | +2025,ai_accelerators,15217,0.5092 | ||||
| 283 | +2025,ai_agents,56916,0.6012 | ||||
| 284 | +2025,ai_safety,69356,0.5776 | ||||
| 285 | +2025,algorithms,228420,0.3291 | ||||
| 286 | +2025,applied_cryptography,98047,0.4293 | ||||
| 287 | +2025,autonomous_scientific_discovery,5478,0.8058 | ||||
| 288 | +2025,blockchain,121711,0.2119 | ||||
| 289 | +2025,cloud_computing,79661,0.4989 | ||||
| 290 | +2025,compilers,4902,0.8413 | ||||
| 291 | +2025,computational_biology,211631,0.2004 | ||||
| 292 | +2025,computational_complexity,25331,0.6998 | ||||
| 293 | +2025,computer_architecture,163238,0.3277 | ||||
| 294 | +2025,computer_graphics,87276,0.4866 | ||||
| 295 | +2025,computer_networks,496882,0.2216 | ||||
| 296 | +2025,computer_vision,1109954,0.1822 | ||||
| 297 | +2025,cybersecurity,435890,0.3327 | ||||
| 298 | +2025,database_systems,120604,0.5151 | ||||
| 299 | +2025,deep_learning_architectures,245500,0.5031 | ||||
| 300 | +2025,distributed_computing,89968,0.6073 | ||||
| 301 | +2025,edge_computing,85425,0.5495 | ||||
| 302 | +2025,federated_learning,13772,0.7339 | ||||
| 303 | +2025,formal_verification,32650,0.4987 | ||||
| 304 | +2025,foundation_models,13678,0.8794 | ||||
| 305 | +2025,generative_ai,82172,0.6386 | ||||
| 306 | +2025,hci,402677,0.2858 | ||||
| 307 | +2025,information_retrieval,121877,0.374 | ||||
| 308 | +2025,llm,343828,0.4589 | ||||
| 309 | +2025,ml_theory,245517,0.4307 | ||||
| 310 | +2025,multimodal_ai,48170,0.7293 | ||||
| 311 | +2025,nlp,324909,0.374 | ||||
| 312 | +2025,operating_systems,9257,0.8159 | ||||
| 313 | +2025,privacy_preserving_computation,66195,0.5222 | ||||
| 314 | +2025,programming_languages,27870,0.6435 | ||||
| 315 | +2025,quantum_computing,133108,0.1459 | ||||
| 316 | +2025,reinforcement_learning,99228,0.3679 | ||||
| 317 | +2025,robotics,269974,0.2587 | ||||
| 318 | +2025,scientific_computing,143714,0.4435 | ||||
| 319 | +2025,software_engineering,267473,0.3336 | ||||
| 320 | +2025,speech_audio,137912,0.2049 | ||||
| 321 | +2025,sustainable_computing,9666,0.7883 | ||||
| 322 | +2026,ai_accelerators,10745,0.5301 | ||||
| 323 | +2026,ai_agents,74557,0.6445 | ||||
| 324 | +2026,ai_safety,75373,0.6246 | ||||
| 325 | +2026,algorithms,192827,0.3229 | ||||
| 326 | +2026,applied_cryptography,67603,0.4474 | ||||
| 327 | +2026,autonomous_scientific_discovery,6582,0.8004 | ||||
| 328 | +2026,blockchain,69117,0.2277 | ||||
| 329 | +2026,cloud_computing,46035,0.5551 | ||||
| 330 | +2026,compilers,4061,0.8527 | ||||
| 331 | +2026,computational_biology,170553,0.1955 | ||||
| 332 | +2026,computational_complexity,24972,0.6565 | ||||
| 333 | +2026,computer_architecture,100060,0.3776 | ||||
| 334 | +2026,computer_graphics,68366,0.486 | ||||
| 335 | +2026,computer_networks,261852,0.2593 | ||||
| 336 | +2026,computer_vision,730253,0.2052 | ||||
| 337 | +2026,cybersecurity,311429,0.3431 | ||||
| 338 | +2026,database_systems,90810,0.5413 | ||||
| 339 | +2026,deep_learning_architectures,172458,0.5203 | ||||
| 340 | +2026,distributed_computing,63582,0.6299 | ||||
| 341 | +2026,edge_computing,47987,0.5856 | ||||
| 342 | +2026,federated_learning,10721,0.752 | ||||
| 343 | +2026,formal_verification,32567,0.5272 | ||||
| 344 | +2026,foundation_models,19258,0.8476 | ||||
| 345 | +2026,generative_ai,84628,0.6465 | ||||
| 346 | +2026,hci,313063,0.303 | ||||
| 347 | +2026,information_retrieval,109436,0.4269 | ||||
| 348 | +2026,llm,372690,0.4594 | ||||
| 349 | +2026,ml_theory,208330,0.4467 | ||||
| 350 | +2026,multimodal_ai,55739,0.7366 | ||||
| 351 | +2026,nlp,242621,0.4251 | ||||
| 352 | +2026,operating_systems,6866,0.8504 | ||||
| 353 | +2026,privacy_preserving_computation,48458,0.5405 | ||||
| 354 | +2026,programming_languages,23414,0.635 | ||||
| 355 | +2026,quantum_computing,140037,0.129 | ||||
| 356 | +2026,reinforcement_learning,86233,0.4415 | ||||
| 357 | +2026,robotics,194894,0.2596 | ||||
| 358 | +2026,scientific_computing,125821,0.4213 | ||||
| 359 | +2026,software_engineering,230827,0.3578 | ||||
| 360 | +2026,speech_audio,109792,0.2175 | ||||
| 361 | +2026,sustainable_computing,6619,0.7892 | ||||
| 362 | |||||