WP44: fixes for audit findings P0-1/P0-2 and P1 (NULL-safe eligibility, PRISMA identity assert, arXiv-id dedup relation, first_date precision, first-author check, citation windows with completeness flags and merged preprint citations, seasonally adjusted Bai-Perron, leakage-free detector cutoffs)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
6 changed files +98 −24
modified
bibliometrics/03_citation_impact.py
+11 −5
@@ -16,12 +16,17 @@ def main(): | ||
| 16 | 16 | out = os.path.join(R, "results/bibliometrics"); os.makedirs(out, exist_ok=True) |
| 17 | 17 | con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb")); con.execute("SET threads TO 20; SET memory_limit='60GB';") |
| 18 | 18 | con.execute("""CREATE OR REPLACE TABLE work_citation_windows AS |
| 19 | − WITH E AS (SELECT id, publication_year AS y, type, primary_subfield_id AS sf, cited_by_count, fwci FROM corpus_works WHERE eligible), | |
| 19 | + WITH E AS (SELECT id, preprint_work_id, publication_year AS y, coalesce(type, 'unknown') AS type, coalesce(primary_subfield_id, -1) AS sf, cited_by_count, fwci | |
| 20 | + FROM corpus_works WHERE eligible AND NOT e6_retracted), | |
| 21 | + -- citations received by the canonical node include those of its merged preprint version (E5) | |
| 22 | + CY AS (SELECT e.id AS work_id, c.year, c.cited_by_count FROM oa_counts_by_year c JOIN E e ON e.id = c.work_id | |
| 23 | + UNION ALL SELECT e.id, c.year, c.cited_by_count FROM oa_counts_by_year c JOIN E e ON e.preprint_work_id = c.work_id), | |
| 20 | 24 | C AS (SELECT c.work_id, sum(CASE WHEN c.year <= e.y + 1 THEN c.cited_by_count ELSE 0 END) AS c12, |
| 21 | 25 | sum(CASE WHEN c.year <= e.y + 2 THEN c.cited_by_count ELSE 0 END) AS c36, |
| 22 | 26 | sum(CASE WHEN c.year <= e.y + 3 THEN c.cited_by_count ELSE 0 END) AS c48 |
| 23 | − FROM oa_counts_by_year c JOIN E e ON e.id = c.work_id GROUP BY 1) | |
| 24 | − SELECT e.id AS work_id, e.y AS year, e.type, e.sf, e.cited_by_count, e.fwci, coalesce(c.c12,0) c12, coalesce(c.c36,0) c36, coalesce(c.c48,0) c48 | |
| 27 | + FROM CY c JOIN E e ON e.id = c.work_id GROUP BY 1) | |
| 28 | + SELECT e.id AS work_id, e.y AS year, e.type, e.sf, e.cited_by_count, e.fwci, coalesce(c.c12,0) c12, coalesce(c.c36,0) c36, coalesce(c.c48,0) c48, | |
| 29 | + (e.y + 1 <= 2025) AS c12_complete, (e.y + 2 <= 2025) AS c36_complete, (e.y + 3 <= 2025) AS c48_complete | |
| 25 | 30 | FROM E e LEFT JOIN C c ON c.work_id = e.id""") |
| 26 | 31 | con.execute("""CREATE OR REPLACE TABLE work_citation_windows2 AS |
| 27 | 32 | SELECT w.*, CASE WHEN m12 > 0 THEN c12 / m12 END AS ncs12, CASE WHEN m36 > 0 THEN c36 / m36 END AS ncs36 |
@@ -29,12 +34,13 @@ def main(): | ||
| 29 | 34 | USING (year, type, sf)""") |
| 30 | 35 | con.execute("DROP TABLE work_citation_windows; ALTER TABLE work_citation_windows2 RENAME TO work_citation_windows") |
| 31 | 36 | con.execute(f"COPY work_citation_windows TO '{R}/data/processed/corpus/work_citation_windows.parquet' (FORMAT PARQUET, COMPRESSION ZSTD)") |
| 32 | − con.execute(f"""COPY (SELECT year, count(*) n, round(avg(c12),3) mean_c12, round(avg(c36),3) mean_c36, quantile_cont(c36,0.5) med_c36, | |
| 37 | + con.execute(f"""COPY (SELECT year, count(*) n, any_value(c12_complete) c12_complete, any_value(c36_complete) c36_complete, round(avg(c12),3) mean_c12, round(avg(c36),3) mean_c36, quantile_cont(c36,0.5) med_c36, | |
| 33 | 38 | round(sum((c36=0)::int)*1.0/count(*),4) share_uncited_36, round(sum((c12=0)::int)*1.0/count(*),4) share_uncited_12, |
| 34 | 39 | quantile_cont(c36,0.99) p99_c36, max(c36) max_c36, |
| 35 | 40 | round((2*sum(c36*rk) / (count(*)*sum(c36)) - (count(*)+1.0)/count(*)),4) gini_c36, |
| 41 | + round((2*sum(c12*rk12) / (count(*)*sum(c12)) - (count(*)+1.0)/count(*)),4) gini_c12, | |
| 36 | 42 | round(avg(fwci),3) mean_fwci, round(quantile_cont(fwci,0.5),3) med_fwci |
| 37 | − FROM (SELECT *, row_number() OVER (PARTITION BY year ORDER BY c36) rk FROM work_citation_windows) GROUP BY 1 ORDER BY 1) TO '{out}/citation_windows_yearly.csv' (HEADER)""") | |
| 43 | + FROM (SELECT *, row_number() OVER (PARTITION BY year ORDER BY c36, work_id) rk, row_number() OVER (PARTITION BY year ORDER BY c12, work_id) rk12 FROM work_citation_windows) GROUP BY 1 ORDER BY 1) TO '{out}/citation_windows_yearly.csv' (HEADER)""") | |
| 38 | 44 | con.execute(f"""COPY (SELECT * FROM (SELECT w.year, w.work_id, c.title, c.source_name, c.type, w.c12, w.c36, w.cited_by_count, w.fwci, w.ncs36, |
| 39 | 45 | row_number() OVER (PARTITION BY w.year ORDER BY w.c36 DESC, w.cited_by_count DESC) rk FROM work_citation_windows w JOIN corpus_works c ON c.id = w.work_id) WHERE rk <= 50 ORDER BY year, rk) TO '{out}/top_cited_by_year.csv' (HEADER)""") |
| 40 | 46 | print(open(f"{out}/citation_windows_yearly.csv").read()) |
modified
econometrics/run_structural_change.py
+7 −4
@@ -25,7 +25,7 @@ def series_categories(R): | ||
| 25 | 25 | if not os.path.exists(p): |
| 26 | 26 | return {} |
| 27 | 27 | d = pd.read_csv(p); col = "n_month_precision" if "n_month_precision" in d.columns else "n_multilabel" |
| 28 | − return {f"cat_{c}": g.set_index("first_month")[col] for c, g in d.groupby("category_id")} | |
| 28 | + return {f"cat_{c}": g.groupby("first_month")[col].sum() for c, g in d.groupby("category_id")} | |
| 29 | 29 | |
| 30 | 30 | |
| 31 | 31 | def run_one(name, s, first, last, out, placebo_baseline_only=True, donut=0): |
@@ -48,10 +48,13 @@ def run_one(name, s, first, last, out, placebo_baseline_only=True, donut=0): | ||
| 48 | 48 | keep = np.abs((dates - pd.Timestamp(d)).days) > donut * 31 |
| 49 | 49 | od, _ = its_fit(y[keep], dates[keep], d); res["its_chatgpt_donut"] = {"months_excluded_each_side": donut, **od} |
| 50 | 50 | pd.DataFrame({"month": idx, "y": s.values, "fitted": np.exp(fit.fittedvalues) - 1}).to_csv(f"{out}/fit_{name}.csv", index=False) |
| 51 | − bp = bai_perron(np.log(y), dates, max_breaks=3); res["bai_perron"] = {k: v for k, v in bp.items() if k != "by_m"} | |
| 51 | + ly = np.log(y); months_of_year = dates.month; base = dates < pd.Timestamp("2022-12-01") | |
| 52 | + seas = {m: ly[base & (months_of_year == m)].mean() - ly[base].mean() for m in range(1, 13)} | |
| 53 | + ly_sa = ly - np.array([seas[m] for m in months_of_year]) # seasonal adjustment from baseline month effects | |
| 54 | + bp = bai_perron(ly_sa, dates, max_breaks=3); res["bai_perron"] = {k: v for k, v in bp.items() if k != "by_m"}; res["bai_perron"]["input"] = "log, seasonally adjusted (baseline month means)" | |
| 52 | 55 | res["bai_perron"]["bic_by_m"] = {m: round(v["bic"], 2) for m, v in bp["by_m"].items()} |
| 53 | − res["pelt"] = [idx[i] for i in pelt(np.log(y)) if i < len(idx)] | |
| 54 | − _, cp, mrl = bocpd(np.log(y)); resets = [idx[t] for t in range(12, len(idx)) if mrl[t] <= 2 and mrl[t - 1] > 12] | |
| 56 | + res["pelt"] = [idx[i] for i in pelt(ly_sa) if i < len(idx)] | |
| 57 | + _, cp, mrl = bocpd(ly_sa); resets = [idx[t] for t in range(12, len(idx)) if mrl[t] <= 2 and mrl[t - 1] > 12] | |
| 55 | 58 | res["bocpd_resets"] = resets; res["bocpd_cp_prob_2022_12_to_2023_03"] = float(cp[[i for i, m in enumerate(idx) if "2022-12" <= m <= "2023-03"]].max()) if any("2022-12" <= m <= "2023-03" for m in idx) else None |
| 56 | 59 | return res |
| 57 | 60 | |
modified
experiments/emerging_detection.py
+4 −3
@@ -75,16 +75,17 @@ def labels_at(P, T_idx, h): | ||
| 75 | 75 | |
| 76 | 76 | |
| 77 | 77 | def main(): |
| 78 | − ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); ap.add_argument("--last-month", default="2026-10") | |
| 78 | + ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); ap.add_argument("--last-month", default="2026-06") | |
| 79 | 79 | ap.add_argument("--horizon", type=int, default=12); a = ap.parse_args(); R = a.root |
| 80 | 80 | out = os.path.join(R, "results/emerging"); os.makedirs(out, exist_ok=True) |
| 81 | 81 | con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb")); con.execute("SET threads TO 20; SET memory_limit='60GB';") |
| 82 | 82 | P, C = build_panel(R, con, a.last_month); P.to_parquet(f"{out}/panel_volume.parquet"); C.to_parquet(f"{out}/panel_citations.parquet") |
| 83 | 83 | months = list(P.columns); h = a.horizon |
| 84 | 84 | def idx(m): return months.index(m) |
| 85 | − last_train = "2022-09" if h <= 12 else "2021-09" # labels of training cutoffs must be fully observed before the first test cutoff | |
| 85 | + # no leakage: every training label must be observable (T + h) BEFORE the first test cutoff (2023-01) | |
| 86 | + last_train = "2021-12" if h <= 12 else "2020-12" | |
| 86 | 87 | train_cuts = [idx(m) for m in months if "2019-12" <= m <= last_train and idx(m) % 3 == 0] |
| 87 | − test_cuts = [idx(m) for m in months if "2023-01" <= m <= "2024-10" and idx(m) % 3 == 0 and idx(m) + h < len(months)] | |
| 88 | + test_cuts = [idx(m) for m in months if "2023-01" <= m <= "2024-10" and idx(m) % 3 == 0 and idx(m) + h < len(months)] # --last-month must be the last COMPLETE month | |
| 88 | 89 | rows = [] |
| 89 | 90 | for T in train_cuts + test_cuts: |
| 90 | 91 | f = features_at(P, C, T); l = labels_at(P, T, h); rows.append(f.merge(l, on=["topic_id", "T_idx"])) |
modified
ingestion/link_sources.py
+4 −1
@@ -56,8 +56,11 @@ def main(): | ||
| 56 | 56 | WHERE tlen >= 25 AND dblp_key NOT IN (SELECT dblp_key FROM link_dblp_oa) GROUP BY ntitle HAVING count(*) = 1""") |
| 57 | 57 | con.execute("""CREATE OR REPLACE TEMP TABLE o_un AS SELECT ntitle, any_value(oa_id) AS oa_id, any_value(year) AS year FROM oa_keys |
| 58 | 58 | WHERE tlen >= 25 AND oa_id NOT IN (SELECT oa_id FROM link_dblp_oa) GROUP BY ntitle HAVING count(*) = 1""") |
| 59 | + con.execute("""CREATE OR REPLACE TEMP TABLE d_fa AS SELECT key AS dblp_key, lower(regexp_extract(authors[1], '([^ ]+)$', 1)) AS fa FROM dblp_raw WHERE len(authors) > 0""") | |
| 60 | + con.execute("""CREATE OR REPLACE TEMP TABLE o_fa AS SELECT work_id AS oa_id, lower(regexp_extract(author_name, '([^ ]+)$', 1)) AS fa FROM oa_authorships WHERE position = 0""") | |
| 59 | 61 | con.execute("""INSERT INTO link_dblp_oa SELECT d.dblp_key, o.oa_id, 'title_year' FROM d_un d JOIN o_un o ON d.ntitle = o.ntitle |
| 60 | − WHERE abs(d.year - o.year) <= 1""") | |
| 62 | + LEFT JOIN d_fa ON d_fa.dblp_key = d.dblp_key LEFT JOIN o_fa ON o_fa.oa_id = o.oa_id | |
| 63 | + WHERE abs(d.year - o.year) <= 1 AND (d_fa.fa IS NULL OR o_fa.fa IS NULL OR d_fa.fa = o_fa.fa)""") | |
| 61 | 64 | log("dblp linked") |
| 62 | 65 | # --- arXiv ↔ OpenAlex --- |
| 63 | 66 | con.execute("""CREATE OR REPLACE TABLE link_arxiv_oa AS |
modified
manuscript/sections/part02_evolution.tex
+46 −0
@@ -143,3 +143,49 @@ Second, each wave shortened the publication cycle and shifted the version of rec | ||
| 143 | 143 | |
| 144 | 144 | Third, the three waves were cumulative in infrastructure and in institutions. The industrial laboratories that produced the web systems trained the generation that produced deep learning, and the frameworks and compute of the deep-learning wave made the transformer era possible. The post-2022 period inherited all of it: the laboratories, the frameworks, the model hubs, the benchmark culture, and the preprint norm. Whatever is new about the period must be identified against that inheritance rather than attributed to a single product release, and the baseline window of 2018--2022 was chosen so that the inheritance is observed directly rather than assumed. |
| 145 | 145 | |
| 146 | +\section{The non-AI areas at the baseline, 2018--2022} | |
| 147 | +\label{sec:p2-baseline} | |
| 148 | + | |
| 149 | +The hypotheses of this study concern not only the growth of AI but the condition of everything else. H2 predicts that at least one non-AI area declines in relative share while growing in absolute terms; H3 predicts that formal verification, computational complexity, operating systems and computer architecture show \emph{no} structural break in the window around November 2022; H4 predicts that citations crossing from those areas into AI increase. Such predictions can be evaluated only if the state of each area at the baseline is described independently of the AI narrative. This section provides that description for the thirty-one non-AI leaves of the ontology, grouped by area, in a deliberately compressed form: for each, the problems that defined the 2018--2022 literature, the landmark results that the corpus's citation data should recover, and the points of contact with machine learning that existed before 2022. Where a result is known to the authors only from a report rather than from a peer-reviewed paper, that is stated. | |
| 150 | + | |
| 151 | +\subsection{Systems, cloud and distributed computing} | |
| 152 | +\label{sec:p2-systems} | |
| 153 | + | |
| 154 | +The systems literature of the baseline period was organised around the consequences of the warehouse-scale computer \citep{barroso2018datacenter}. Three agendas dominated. The first was the \emph{serverless} abstraction, in which the unit of deployment is a stateless function and the provider manages everything else; the Berkeley view presented it as the next step in the simplification of cloud programming \citep{jonas2019serverless}, while a contemporaneous critique from the database community argued that its restrictions on state, communication and hardware access made it a step backward for data-intensive work \citep{hellerstein2019serverless}. The disagreement was productive: much of the 2019--2022 systems literature consists of attempts to give serverless platforms the state and the locality they lacked. The second agenda was \emph{resource disaggregation}, the separation of compute, memory and storage into independently scaled pools connected by fast networks, an idea with roots in blade-server memory expansion \citep{lim2009disaggregated} that became a central topic of operating-systems research once network latency fell within an order of magnitude of memory latency. The third was \emph{tail latency}: in a service composed of thousands of components, rare slow responses in any of them dominate the user-visible distribution, and the techniques of hedged requests, replication and tied requests described by \citet{dean2013tail} became the standard vocabulary of large-scale service design. | |
| 155 | + | |
| 156 | +Operating systems research in the period was marked by the consequences of the Spectre and Meltdown disclosures, discussed under security below, which forced kernel designers to reconsider the cost of isolation, and by the growth of in-kernel programmability. Distributed computing as a theory remained focused on consensus, fault tolerance and the coordination of replicated state, with blockchain systems supplying new adversarial models (Section~\ref{sec:p2-security}). The contact with machine learning before 2022 was of two kinds: ``systems for ML'', the training and serving infrastructure of Section~\ref{sec:p2-transformer}, which systems venues absorbed readily, and ``ML for systems'', the use of learned models for scheduling, configuration and resource management, which was an active but minority topic. The ontology treats cloud and edge computing as children of distributed computing precisely because the venue structure does: OSDI, SOSP, EuroSys and NSDI publish all three. | |
| 157 | + | |
| 158 | +\subsection{Hardware and computer architecture} | |
| 159 | +\label{sec:p2-hardware} | |
| 160 | + | |
| 161 | +The baseline period in computer architecture was dominated by the end of two scaling regimes. Dennard scaling, the observation that transistor power density stays constant as transistors shrink \citep{dennard1974scaling}, had ended in the mid-2000s; its consequence, that a growing fraction of a chip must be left dark at any time, was quantified by \citet{esmaeilzadeh2011dark} and set the agenda for the following decade. The slowing of Moore's law completed the picture. The Turing lecture of \citet{hennessy2019golden} drew the conclusion that general-purpose performance gains were over and that the future lay in domain-specific architectures, open instruction sets and agile hardware design; \citet{leiserson2020room} argued in parallel that performance would henceforth come from software, algorithms and hardware specialisation rather than from the ``bottom'' of the stack. The tensor processing unit \citep{jouppi2017tpu} is the canonical example of the domain-specific architecture, and its publication in a mainstream architecture venue is a measure of how completely machine learning had become the architecture community's defining workload by 2017. AI accelerators are a child of computer architecture in the ontology for that reason, and the hypothesis that the architecture category shows no post-2022 break is, in effect, the hypothesis that this absorption was complete before ChatGPT. | |
| 162 | + | |
| 163 | +Compilers underwent a parallel reorganisation around machine-learning workloads. The multi-level intermediate representation MLIR \citep{lattner2021mlir}, designed to let one compiler infrastructure serve domain-specific languages, tensor programs and hardware targets alike, and the end-to-end tensor compiler TVM \citep{chen2018tvm} represent the state of the art at the baseline. Both treat the deep-learning graph as a first-class compilation target; neither depends on large language models. | |
| 164 | + | |
| 165 | +\subsection{Theory of computation} | |
| 166 | +\label{sec:p2-theory} | |
| 167 | + | |
| 168 | +Two developments characterise theoretical computer science at the baseline, one internal and one at the boundary with machine learning. | |
| 169 | + | |
| 170 | +The internal development is \emph{fine-grained complexity}: the programme of explaining why polynomial-time problems such as edit distance, all-pairs shortest paths and orthogonal vectors have resisted improvement for decades, by proving that faster algorithms for one would contradict a small number of hardness conjectures, above all the strong exponential-time hypothesis \citep{vassilevska2015hardness,williams2018finegrained}. The programme reorganised algorithms research around conditional lower bounds in the same way that NP-completeness had organised it around unconditional intractability, and its vocabulary is a reliable marker of the complexity category in this corpus. | |
| 171 | + | |
| 172 | +The boundary development is the \emph{theory of deep learning}. The empirical observation that networks with enough parameters to memorise random labels nevertheless generalise \citep{zhang2021rethinking} undermined the classical uniform-convergence account of generalisation and prompted new theory: the double-descent curve, which reconciles the classical bias--variance trade-off with the behaviour of overparameterised models \citep{belkin2019reconciling}, and the neural tangent kernel, which characterises the training dynamics of infinitely wide networks \citep{jacot2018ntk}. The machine-learning-theory leaf of the ontology houses this literature, which belongs sociologically to the learning conferences and intellectually to the theory community. Its presence is a reminder that the theory area was in contact with AI long before 2022, through learning theory, optimisation and the complexity of sampling and inference. | |
| 173 | + | |
| 174 | +\subsection{Programming languages, compilers and formal verification} | |
| 175 | +\label{sec:p2-pl} | |
| 176 | + | |
| 177 | +The programming-languages literature of the baseline is marked by the success of a language. Rust, whose ownership type system gives memory safety without garbage collection \citep{matsakis2014rust}, became the first systems language to be adopted at scale on the strength of a safety guarantee; the RustBelt project supplied the semantic foundation by proving, in a separation-logic framework mechanised in Coq, that the safe fragment is sound and that the unsafe library code on which it rests respects the invariants \citep{jung2017rustbelt}. The result is characteristic of the period's formal-methods culture: a verification effort that follows an engineering artefact rather than preceding it, and that uses a proof assistant as its medium. | |
| 178 | + | |
| 179 | +Proof assistants themselves reached a new stage. The verified C compiler CompCert \citep{leroy2009compcert} and the verified microkernel seL4 \citep{klein2009sel4} had shown a decade earlier that production software could be proved correct; the Kepler conjecture was formally proved in 2017 after a twelve-year effort \citep{hales2017kepler}; and the Lean theorem prover \citep{demoura2015lean}, re-implemented as a programming language in its fourth version \citep{demoura2021lean4}, acquired a community-maintained mathematical library whose scale and organisation were new in formalised mathematics \citep{mathlib2020}. The relevance to the post-2022 period is direct: Lean and its library became the environment in which language models were later asked to prove theorems, so that formal verification, one of the categories predicted \emph{not} to break in H3, is also one where the contact with AI is most visible in the recent literature. The hypothesis is therefore a sharp one. | |
| 180 | + | |
| 181 | +Software engineering had already begun to absorb learned models before 2022. The survey of \citet{allamanis2018survey} organised a decade of ``big code'' research around the naturalness hypothesis, that source code is as statistically regular as natural language; the CodeXGLUE benchmark collection \citep{lu2021codexglue} standardised evaluation; and the Codex models \citep{chen2021codex} made program synthesis from natural language a product. The software-engineering category is thus expected to show an early and strong AI signal, and its baseline is not an AI-free one. | |
| 182 | + | |
| 183 | +\subsection{Security and cryptography} | |
| 184 | +\label{sec:p2-security} | |
| 185 | + | |
| 186 | +Three events organise the security and cryptography literature of the baseline. The first is the disclosure, in January 2018, of Spectre and Meltdown, which showed that speculative execution in essentially all high-performance processors leaks data across isolation boundaries \citep{kocher2019spectre,lipp2020meltdown}. The disclosures created a sustained literature on microarchitectural side channels and their mitigations that cuts across security, architecture and operating systems, and they illustrate the kind of cross-area citation flow that H4 measures. | |
| 187 | + | |
| 188 | +The second is the standardisation of \emph{post-quantum cryptography}. Shor's algorithm \citep{shor1997algorithms} had made the eventual obsolescence of RSA and elliptic-curve schemes a matter of engineering timelines, and the case for migrating before large quantum computers exist was made to a general scientific audience by \citet{bernstein2017pqc}. The process run by the United States National Institute of Standards and Technology from 2016 selected lattice-based and hash-based schemes, published as federal standards in August 2024 \citep{nist2024fips203,nist2024fips204,nist2024fips205}. The applied-cryptography category of this corpus therefore contains a large and growing body of work whose dynamics are driven by a standards calendar entirely independent of language models, which makes it a useful placebo field. Fully homomorphic encryption, shown to be possible by \citet{gentry2009fhe}, remained the subject of a steady literature on efficiency. | |
| 189 | + | |
| 190 | +The third is the maturation of \emph{adversarial machine learning}. The observation that imperceptible perturbations can change a network's output \citep{szegedy2014intriguing} and the attacks and evaluation standards that followed \citep{carlini2017robustness} created, by 2018, a field with ten years of history and a recognised research agenda \citep{biggio2018wild}. The ontology scores adversarial work in both cybersecurity and AI safety, which reflects the baseline state: the problem belongs to both communities and neither owns it. Blockchain systems, finally, entered the period with a systematised research agenda \citep{bonneau2015sok} and left it as a mature if contested area whose publication dynamics follow cryptocurrency markets more than academic calendars. | |
| 191 | + | |
modified
systematic_review/define_corpus.py
+26 −11
@@ -28,18 +28,25 @@ def main(): | ||
| 28 | 28 | (w.id IN (SELECT oa_id FROM link_acl_oa)) AS in_acl, |
| 29 | 29 | x.arxiv_id AS arxiv_id_linked, x.created AS arxiv_created, x.primary_category AS arxiv_primary, |
| 30 | 30 | (x.primary_category LIKE 'cs.%') AS arxiv_cs_primary, |
| 31 | − (w.doi LIKE '10.48550/arxiv.%') AS is_arxiv_doi | |
| 31 | + coalesce(w.doi LIKE '10.48550/arxiv.%', FALSE) AS is_arxiv_doi | |
| 32 | 32 | FROM oa_works w |
| 33 | 33 | LEFT JOIN (SELECT l.oa_id, any_value(k.arxiv_id) arxiv_id, min(k.created) created, any_value(k.primary_category) primary_category |
| 34 | 34 | FROM link_arxiv_oa l JOIN arxiv_keys k USING(arxiv_id) GROUP BY l.oa_id) x ON x.oa_id = w.id""") |
| 35 | 35 | log(f"cw0 {n('SELECT count(*) FROM cw0'):,}") |
| 36 | 36 | # ---- S2: preprint ↔ published duplicates (same normalised title, |year| <= 2, one preprint-like, one not) ---- |
| 37 | − con.execute("""CREATE OR REPLACE TABLE dup_pairs AS | |
| 38 | − WITH pre AS (SELECT id, ntitle, publication_year y, publication_date d FROM cw0 WHERE (type = 'preprint' OR is_arxiv_doi) AND length(ntitle) >= 25), | |
| 39 | − pub AS (SELECT id, ntitle, publication_year y, publication_date d FROM cw0 WHERE type <> 'preprint' AND NOT is_arxiv_doi AND length(ntitle) >= 25) | |
| 40 | − SELECT p.id AS preprint_id, b.id AS published_id, p.d AS preprint_date, b.d AS published_date, | |
| 41 | − row_number() OVER (PARTITION BY b.id ORDER BY p.d, p.id) AS rn_pub, row_number() OVER (PARTITION BY p.id ORDER BY b.d, b.id) AS rn_pre | |
| 42 | − FROM pre p JOIN pub b ON p.ntitle = b.ntitle AND abs(p.y - b.y) <= 2""") | |
| 37 | + # published twins must themselves pass E1–E4 (otherwise the preprint stays as the canonical record) | |
| 38 | + con.execute(f"""CREATE OR REPLACE TABLE dup_pairs AS | |
| 39 | + WITH pre AS (SELECT id, ntitle, publication_year y, publication_date d, coalesce(arxiv_id_linked, arxiv_id) AS ax FROM cw0 WHERE (coalesce(type,'') = 'preprint' OR is_arxiv_doi) AND length(ntitle) >= 25), | |
| 40 | + pub AS (SELECT id, ntitle, publication_year y, publication_date d, coalesce(arxiv_id_linked, arxiv_id) AS ax FROM cw0 | |
| 41 | + WHERE coalesce(type,'') <> 'preprint' AND NOT is_arxiv_doi AND length(ntitle) >= 25 | |
| 42 | + AND publication_date BETWEEN '{WINDOW_START}' AND '{WINDOW_END}' AND NOT is_xpac AND title IS NOT NULL | |
| 43 | + AND (coalesce(type,'') IN {KEEP_TYPES} OR in_dblp)), | |
| 44 | + cand AS (SELECT p.id AS preprint_id, b.id AS published_id, p.d AS preprint_date, b.d AS published_date, 'title_year' AS relation | |
| 45 | + FROM pre p JOIN pub b ON p.ntitle = b.ntitle AND abs(p.y - b.y) <= 2 | |
| 46 | + UNION | |
| 47 | + SELECT p.id, b.id, p.d, b.d, 'arxiv_id' FROM pre p JOIN pub b ON p.ax = b.ax AND p.ax IS NOT NULL AND p.id <> b.id) | |
| 48 | + SELECT *, row_number() OVER (PARTITION BY published_id ORDER BY preprint_date, preprint_id) AS rn_pub, | |
| 49 | + row_number() OVER (PARTITION BY preprint_id ORDER BY published_date, published_id) AS rn_pre FROM cand""") | |
| 43 | 50 | con.execute("DELETE FROM dup_pairs WHERE rn_pub > 1 OR rn_pre > 1") |
| 44 | 51 | log(f"duplicate preprint→published pairs: {n('SELECT count(*) FROM dup_pairs'):,}") |
| 45 | 52 | # ---- S3: eligibility flags ---- |
@@ -47,13 +54,17 @@ def main(): | ||
| 47 | 54 | SELECT c.*, |
| 48 | 55 | dp.published_id AS duplicate_of, |
| 49 | 56 | dp2.preprint_id AS preprint_work_id, |
| 50 | − least(c.publication_date, coalesce(dp2.preprint_date, c.publication_date), coalesce(c.arxiv_created, c.publication_date)) AS first_date, | |
| 57 | + -- first public appearance: precise dates win over year-only (YYYY-01-01) publication dates | |
| 58 | + CASE WHEN substr(c.publication_date, 6, 5) = '01-01' AND coalesce(dp2.preprint_date, c.arxiv_created) IS NOT NULL | |
| 59 | + THEN least(coalesce(dp2.preprint_date, c.arxiv_created), coalesce(c.arxiv_created, dp2.preprint_date)) | |
| 60 | + ELSE least(c.publication_date, coalesce(dp2.preprint_date, c.publication_date), coalesce(c.arxiv_created, c.publication_date)) END AS first_date, | |
| 61 | + dp2.relation AS version_relationship, | |
| 51 | 62 | (c.publication_date IS NULL OR c.publication_date < '{WINDOW_START}' OR c.publication_date > '{WINDOW_END}') AS e1_outside_window, |
| 52 | − (c.type NOT IN {KEEP_TYPES} AND NOT (c.in_dblp AND c.type NOT IN {HARD_EXCLUDE_TYPES})) AS e2_type, | |
| 63 | + (coalesce(c.type, '') NOT IN {KEEP_TYPES} AND NOT (c.in_dblp AND coalesce(c.type, '') NOT IN {HARD_EXCLUDE_TYPES})) AS e2_type, | |
| 53 | 64 | (c.is_xpac AND NOT c.in_dblp) AS e3_xpac, |
| 54 | 65 | (c.is_xpac AND c.in_dblp) AS xpac_kept_dblp, |
| 55 | − (c.source_name IN {GENERIC_DEPOSIT_SOURCES} AND NOT c.in_dblp AND c.arxiv_id_linked IS NULL) AS e7_generic_deposit, | |
| 56 | − (CASE WHEN substr(c.publication_date, 6, 5) = '01-01' THEN 'year' ELSE 'month' END) AS date_precision, | |
| 66 | + (coalesce(c.source_name, '') IN {GENERIC_DEPOSIT_SOURCES} AND NOT c.in_dblp AND c.arxiv_id_linked IS NULL) AS e7_generic_deposit, | |
| 67 | + (CASE WHEN substr(c.publication_date, 6, 5) = '01-01' AND coalesce(dp2.preprint_date, c.arxiv_created) IS NULL THEN 'year' ELSE 'month' END) AS date_precision, | |
| 57 | 68 | (c.title IS NULL OR length(trim(c.title)) = 0) AS e4_no_title, |
| 58 | 69 | (dp.published_id IS NOT NULL) AS e5_duplicate, |
| 59 | 70 | c.is_retracted AS e6_retracted, |
@@ -67,6 +78,8 @@ def main(): | ||
| 67 | 78 | con.execute(f"""ALTER TABLE corpus_works ADD COLUMN period VARCHAR; UPDATE corpus_works SET period = CASE |
| 68 | 79 | WHEN first_date < '{WINDOW_START}' THEN 'pre' WHEN first_date < '{BOUNDARY}' THEN 'baseline' WHEN first_date <= '{WINDOW_END}' THEN 'post' ELSE 'future' END""") |
| 69 | 80 | con.execute("ALTER TABLE corpus_works ADD COLUMN first_month VARCHAR; UPDATE corpus_works SET first_month = substr(first_date, 1, 7)") |
| 81 | + assert n("SELECT count(*) FROM corpus_works WHERE eligible IS NULL") == 0, "NULL eligibility flags" | |
| 82 | + assert n("SELECT count(*) FROM corpus_works WHERE first_date IS NULL AND publication_date IS NOT NULL") == 0, "NULL first_date" | |
| 70 | 83 | log("corpus_works built") |
| 71 | 84 | # ---- DBLP-only records (no OpenAlex record after both passes) ---- |
| 72 | 85 | con.execute("""CREATE OR REPLACE TABLE dblp_only_counts AS |
@@ -88,6 +101,8 @@ def main(): | ||
| 88 | 101 | for k in ["e1_outside_window", "e2_type", "e3_xpac", "e4_no_title", "e5_duplicate", "e7_generic_deposit"]: |
| 89 | 102 | P["excluded_sequential"][k] = n(f"SELECT count(*) FROM corpus_works WHERE {cond} AND {k}"); cond += f" AND NOT {k}" |
| 90 | 103 | P["after_exclusions"] = n(f"SELECT count(*) FROM corpus_works WHERE {cond}") |
| 104 | + assert P["identified"]["openalex_total_records_loaded"] == sum(P["excluded_sequential"].values()) + P["after_exclusions"], "PRISMA identity violated" | |
| 105 | + P["version_relationship"] = dict(q("SELECT version_relationship, count(*) FROM corpus_works WHERE eligible AND version_relationship IS NOT NULL GROUP BY 1")) | |
| 91 | 106 | P["cs_core_eligible"] = n("SELECT count(*) FROM corpus_works WHERE eligible") |
| 92 | 107 | P["cs_penumbra_eligible_otherwise"] = n(f"SELECT count(*) FROM corpus_works WHERE {cond} AND cs_penumbra") |
| 93 | 108 | P["eligible_by_period"] = dict(q("SELECT period, count(*) FROM corpus_works WHERE eligible GROUP BY 1")) |
| 94 | 109 | |