WP44: LaTeX table generator, API increment ingestion
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
5 changed files +181 −2
added
ingestion/ingest_api_increment.py
+31 −0
@@ -0,0 +1,31 @@ | ||
| 1 | +#!/usr/bin/env python3 | |
| 2 | +"""WP44 — convert the OpenAlex API increment pages (works with a CS topic published 2026-10-01 → 2026-11-02, fetched after the | |
| 3 | +2026-10-14 snapshot) into the same JSONL.zst part format as the snapshot filter, so that build_corpus.py can normalise them: | |
| 4 | +data/raw/openalex/works_pass3/cs/updated_date=api-increment__part_0000.jsonl.zst. Works already present in pass 1 or pass 2 | |
| 5 | +are skipped (ids read from the compact works tables). Provenance: api_increment/ledger.json (calls, cost, slices).""" | |
| 6 | +import argparse, glob, json, os, duckdb, orjson, zstandard as zstd | |
| 7 | + | |
| 8 | + | |
| 9 | +def main(): | |
| 10 | + ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); a = ap.parse_args(); R = a.root | |
| 11 | + con = duckdb.connect(); known = set() | |
| 12 | + for f in ("works", "works_p2"): | |
| 13 | + p = f"{R}/data/processed/compact/{f}.parquet" | |
| 14 | + if os.path.exists(p): | |
| 15 | + known.update(x[0] for x in con.execute(f"SELECT id FROM read_parquet('{p}')").fetchall()) | |
| 16 | + out_dir = f"{R}/data/raw/openalex/works_pass3/cs"; os.makedirs(out_dir, exist_ok=True) | |
| 17 | + seen = set(); n_pages = n_new = n_dup = 0 | |
| 18 | + with open(f"{out_dir}/updated_date=api-increment__part_0000.jsonl.zst", "wb") as fh, zstd.ZstdCompressor(level=6).stream_writer(fh) as w: | |
| 19 | + for pf in sorted(glob.glob(f"{R}/data/raw/openalex/api_increment/pages/*/page_*.json")): | |
| 20 | + n_pages += 1 | |
| 21 | + for work in json.load(open(pf)).get("results", []): | |
| 22 | + wid = int(work["id"].rsplit("W", 1)[-1]) | |
| 23 | + if wid in known or wid in seen: | |
| 24 | + n_dup += 1; continue | |
| 25 | + seen.add(wid); n_new += 1; w.write(orjson.dumps(work) + b"\n") | |
| 26 | + rep = {"pages": n_pages, "new_works": n_new, "already_in_snapshot": n_dup, "ledger": json.load(open(f"{R}/data/raw/openalex/api_increment/ledger.json"))} | |
| 27 | + json.dump(rep, open(f"{R}/data/raw/openalex/works_pass3/increment_report.json", "w"), indent=1, default=str); print({k: v for k, v in rep.items() if k != "ledger"}) | |
| 28 | + | |
| 29 | + | |
| 30 | +if __name__ == "__main__": | |
| 31 | + main() | |
modified
manuscript/sections/part02_evolution.tex
+24 −2
@@ -143,6 +143,19 @@ Second, each wave shortened the publication cycle and shifted the version of rec | ||
| 143 | 143 | |
| 144 | 144 | Third, the three waves were cumulative in infrastructure and in institutions. The industrial laboratories that produced the web systems trained the generation that produced deep learning, and the frameworks and compute of the deep-learning wave made the transformer era possible. The post-2022 period inherited all of it: the laboratories, the frameworks, the model hubs, the benchmark culture, and the preprint norm. Whatever is new about the period must be identified against that inheritance rather than attributed to a single product release, and the baseline window of 2018--2022 was chosen so that the inheritance is observed directly rather than assumed. |
| 145 | 145 | |
| 146 | +\subsection{The publication system as it stood in 2022} | |
| 147 | +\label{sec:p2-pubsystem} | |
| 148 | + | |
| 149 | +The three waves left computer science, on the eve of the reference date, with a publication system unlike that of any other science, and the system's properties determine what the corpus can and cannot see. Four of them are load-bearing for the analyses that follow. | |
| 150 | + | |
| 151 | +The first is the dominance of the preprint. By 2022 the arXiv was the first and often the only public version of most machine-learning results, and a growing share of systems, theory and security results as well; the matching study of \citet{lin2020preprints} found that a majority of computer-science preprints in their sample could not be matched to a published version, and that the matched ones had been published with a median delay of many months. The consequence is that the arXiv series lead the indexed series, and that a corpus which dates works by their indexed publication date will place a change later than it occurred and will smear it over the publication lag. Part~III dates each work by the earliest of its preprint and publication dates when both are known, and Part~IX reports the breaks on both clocks. | |
| 152 | + | |
| 153 | +The second is the conference calendar. The major venues of every area have fixed annual deadlines, and preprint postings cluster in the weeks before and after them; monthly series of computer-science output are therefore strongly seasonal, with a pattern that differs by area because the calendars differ. Seasonality is not a nuisance to be removed once and forgotten; a change in the \emph{amplitude} of seasonality, for instance because a venue grew or because preprints began to be posted without regard to deadlines, is itself a structural change, and the seasonal component $\mathbf{s}_c(t)$ of equation~\eqref{eq:p2-its} is modelled separately for the baseline and the research period for that reason. | |
| 154 | + | |
| 155 | +The third is the state of peer review. The volume of submissions to the leading machine-learning conferences had grown by an order of magnitude in a decade; reviewer pools had not. The debate about whether conferences remained an adequate venue of record \citep{vardi2009conferences,fortnow2009growup} was, by 2022, less about prestige than about capacity, and the reproducibility programmes of \citet{pineau2021reproducibility} were one institutional response. The relevance to this study is twofold. Reviewing capacity constrains the number of works that can appear in venue-defined categories, so that growth under capacity constraints shows up as preprint growth and as the founding of new venues rather than as growth in existing streams; and the reviews themselves became, after 2022, one of the first places where the use of language models was measured \citep{liang2024monitoring}. | |
| 156 | + | |
| 157 | +The fourth is the asymmetry between what is published and what is released. The model reports of Section~\ref{sec:p2-transformer} describe systems whose weights, data and code are variably available; the artefact-evaluation and badging practices adopted by systems and software-engineering venues during the period had no counterpart at the frontier of AI, where the most consequential systems were the least documented. The reproducibility indicators of RQ9, which the corpus measures through linked code repositories and data availability statements, are therefore expected to differ sharply by category, and a decline in the AI categories would be consistent with the literature rather than anomalous. | |
| 158 | + | |
| 146 | 159 | \section{The non-AI areas at the baseline, 2018--2022} |
| 147 | 160 | \label{sec:p2-baseline} |
| 148 | 161 | |
@@ -208,6 +221,13 @@ Human--computer interaction had, by 2019, a research programme on interaction wi | ||
| 208 | 221 | |
| 209 | 222 | Computer graphics underwent a transformation of its own in 2020, when neural radiance fields showed that a multilayer perceptron could represent a scene well enough to synthesise photorealistic novel views from a handful of photographs \citep{mildenhall2020nerf}. Neural rendering had been surveyed as an emerging area in the same year \citep{tewari2020neural}; within three years, explicit Gaussian primitives optimised by gradient descent had matched the quality of radiance fields at real-time frame rates \citep{kerbl2023gaussian}. The sequence is instructive for the detection methods of Part~IV: neural radiance fields are one of the pre-registered ``known emergence'' topics against which the emerging-topic detector is validated (H8), and they emerged in a non-AI category by the adoption of an AI method, which is exactly the pattern the detector must distinguish from vocabulary drift. |
| 210 | 223 | |
| 224 | +\subsection{Perception, language, speech and robotics} | |
| 225 | +\label{sec:p2-perception} | |
| 226 | + | |
| 227 | +Four leaves of the perception area lie outside the AI area in the ontology while being the historical home of deep learning, and their baseline is therefore the baseline of the deep-learning wave itself. Computer vision had, by 2022, completed the transition from convolutional to attention-based architectures and from supervised to self-supervised pretraining described in Section~\ref{sec:p2-transformer}; its venues (CVPR, ICCV, ECCV) were among the largest in the discipline, and its vocabulary was already that of foundation models. Natural language processing is the subfield in which the transformer era began, and its baseline is paradoxical: the methods that defined the field in 2018, task-specific architectures, feature engineering, and the pipeline of tagging, parsing and semantic analysis, had been displaced by pretrained models before 2022, so that the post-2022 question for NLP is whether the field continued to exist as a distinct community once its central method became everyone's method. The ontology keeps NLP as a task-framed leaf distinct from large language models for that reason, and the ratio of the two is one of the quantities Part~IX reports. | |
| 228 | + | |
| 229 | +Speech and audio computing had followed the same trajectory a few years behind: hybrid acoustic models \citep{hinton2012speech} gave way to end-to-end sequence models and then to self-supervised pretraining on unlabelled audio \citep{baevski2020wav2vec}, and the weakly supervised Whisper models trained on hundreds of thousands of hours of transcribed speech \citep{radford2023whisper} arrived in the same months as ChatGPT. Robotics is the slowest of the four, because physical data are expensive and simulation transfers imperfectly. End-to-end learned visuomotor policies had been demonstrated in 2016 \citep{levine2016visuomotor}, and reinforcement learning in simulation was an established research line, but most of the field's work in the baseline period remained in planning, control and perception for specific platforms. The vision-language-action models of 2023 \citep{brohan2023rt2}, which transfer web-scale pretraining to manipulation, are the point at which the foundation-model recipe reached robotics, and they fall inside the research period; robotics is therefore one of the categories where a late break, rather than one near the reference date, would be the expected signature of the transformation. | |
| 230 | + | |
| 211 | 231 | \subsection{Quantum computing} |
| 212 | 232 | \label{sec:p2-quantum} |
| 213 | 233 | |
@@ -242,14 +262,14 @@ ChatGPT was one release in a dense sequence, and the sequence matters because an | ||
| 242 | 262 | \caption{Candidate reference dates around November 2022. ``Document'' is the verified primary source; dates are those of the first public version of the document or of the release it describes, as given in the source. The list is restricted to items cited in this Part and is not a complete chronology.} |
| 243 | 263 | \label{tab:p2-timeline} |
| 244 | 264 | \small |
| 245 | −\begin{tabularx}{\textwidth}{@{}llX@{}} | |
| 265 | +\begin{tabularx}{\textwidth}{@{}l>{\raggedright\arraybackslash}p{0.34\textwidth}X@{}} | |
| 246 | 266 | \toprule |
| 247 | 267 | Date & Release or result & Document \\ |
| 248 | 268 | \midrule |
| 249 | 269 | 2022-03 & Compute-optimal scaling (Chinchilla) & \citet{hoffmann2022chinchilla}, preprint \\ |
| 250 | 270 | 2022-04 & PaLM; DALL-E 2; Flamingo & \citet{chowdhery2022palm,ramesh2022dalle2,alayrac2022flamingo}, preprints \\ |
| 251 | 271 | 2022-05 & OPT; Gato; Imagen & \citet{zhang2022opt,reed2022gato,saharia2022imagen}, preprints \\ |
| 252 | −2022-06 & Latent diffusion published (CVPR); Stable Diffusion released openly in August & \citet{rombach2022ldm} \\ | |
| 272 | +2022-06 & Latent diffusion (CVPR); open release as Stable Diffusion in August & \citet{rombach2022ldm} \\ | |
| 253 | 273 | 2022-10 & ReAct & \citet{yao2023react}, preprint \\ |
| 254 | 274 | 2022-11 & BLOOM; ChatGPT (30 November) & \citet{bigscience2022bloom}, preprint; \citet{openai2022chatgpt}, web page \\ |
| 255 | 275 | 2022-12 & Whisper & \citet{radford2023whisper}, preprint \\ |
@@ -303,6 +323,8 @@ The most direct evidence of a change in the literature after 2022 is a change in | ||
| 303 | 323 | |
| 304 | 324 | For this study the finding is a measurement problem before it is a result. The classification of Section~\ref{sec:p2-ontology} and the embedding methods of Part~IV both read text, and text that is partly produced by a model trained on the pre-2023 literature is, in a precise sense, less informative about what is new in a paper. Two responses are built into the design. The venue and arXiv signals of equation~\eqref{eq:p2-score} are immune to prose; and the robustness checks of Appendix~D re-estimate the main breaks with classifications that use no keyword signals, so that a break present in the keyword-based series and absent in the venue-based one is flagged as possibly linguistic. |
| 305 | 325 | |
| 326 | +The measurement of LLM-generated text in \emph{peer reviews} deserves separate mention because it bears on the quality of the venue signal. \citet{liang2024monitoring} estimated that a non-trivial share of the reviews submitted to several 2024 machine-learning conferences had been substantially modified by a language model, with the share higher among reviews submitted close to the deadline and among reviewers who did not respond to author rebuttals. If reviewing is partly delegated to models, then acceptance at a selective venue, which the ontology treats as the strongest community signal, is partly a judgement rendered by the models under study. The corpus cannot measure this directly; it is recorded here as a limitation of the venue signal in the research period that does not apply to the baseline. | |
| 327 | + | |
| 306 | 328 | \subsection{Illusions of understanding and the epistemology of AI in science} |
| 307 | 329 | \label{sec:p2-claim-epistemic} |
| 308 | 330 | |
added
manuscript/tables/TABLES.json
+11 −0
@@ -0,0 +1,11 @@ | ||
| 1 | +[ | |
| 2 | + { | |
| 3 | + "file": "tab_linkage.tex", | |
| 4 | + "caption": "Record linkage between sources.", | |
| 5 | + "label": "tab:linkage", | |
| 6 | + "sources": [ | |
| 7 | + "data/metadata/linkage_report.json" | |
| 8 | + ], | |
| 9 | + "rows": 12 | |
| 10 | + } | |
| 11 | +] | |
| \ No newline at end of file | ||
added
manuscript/tables/tab_linkage.tex
+22 −0
@@ -0,0 +1,22 @@ | ||
| 1 | +% generated by tables/make_tables.py from data/metadata/linkage_report.json | |
| 2 | +\begin{table}[htbp]\centering\caption{Record linkage between sources.}\label{tab:linkage} | |
| 3 | +\begin{threeparttable}\small | |
| 4 | +\begin{tabular}{lr} | |
| 5 | +\toprule | |
| 6 | +\textbf{Quantity} & \textbf{Count} \\ | |
| 7 | +\midrule | |
| 8 | +DBLP records 2018–2026 & 4{,}347{,}365 \\ | |
| 9 | +… with DOI & 4{,}052{,}326 \\ | |
| 10 | +DBLP ↔ OpenAlex links by DOI & 2{,}562{,}431 \\ | |
| 11 | +… by normalised title + year (+ first author) & 179{,}990 \\ | |
| 12 | +DBLP records linked (distinct) & 2{,}736{,}903 \\ | |
| 13 | +DBLP records without OpenAlex record & 1{,}610{,}462 \\ | |
| 14 | +arXiv cs records created 2018–2026 & 890{,}852 \\ | |
| 15 | +arXiv ↔ OpenAlex by arXiv id & 792{,}672 \\ | |
| 16 | +… by DOI & 520 \\ | |
| 17 | +… by title + year & 2{,}143 \\ | |
| 18 | +ACL entries 2018–2026 & 85{,}625 \\ | |
| 19 | +ACL ↔ OpenAlex linked (2018–2026) & 72{,}223 \\ | |
| 20 | +\bottomrule | |
| 21 | +\end{tabular} | |
| 22 | +\end{threeparttable}\end{table} | |
added
tables/make_tables.py
+93 −0
@@ -0,0 +1,93 @@ | ||
| 1 | +#!/usr/bin/env python3 | |
| 2 | +"""WP44 — generate LaTeX tables (booktabs) from result files into manuscript/tables/*.tex. Every table is regenerated from | |
| 3 | +CSV/JSON artefacts; the inventory manuscript/tables/TABLES.json records source files and captions for Appendix G.""" | |
| 4 | +import json, os, sys, csv, glob | |
| 5 | +import pandas as pd | |
| 6 | +R = sys.argv[1] if len(sys.argv) > 1 else os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| 7 | +OUT = os.path.join(R, "manuscript/tables"); os.makedirs(OUT, exist_ok=True); INV = [] | |
| 8 | + | |
| 9 | + | |
| 10 | +def tex_escape(s): | |
| 11 | + return str(s).replace("&", "\\&").replace("%", "\\%").replace("_", "\\_").replace("#", "\\#") | |
| 12 | + | |
| 13 | + | |
| 14 | +def write(name, df, caption, label, sources, fmt=None, col_names=None, longtable=False, note=None): | |
| 15 | + df = df.copy() | |
| 16 | + if col_names: df.columns = col_names | |
| 17 | + for c in df.columns: | |
| 18 | + if pd.api.types.is_float_dtype(df[c]): | |
| 19 | + df[c] = df[c].map(lambda v: "" if pd.isna(v) else (fmt or {}).get(c, "{:,.3f}").format(v).replace(",", "{,}")) | |
| 20 | + elif pd.api.types.is_integer_dtype(df[c]): | |
| 21 | + df[c] = df[c].map(lambda v: f"{v:,}".replace(",", "{,}")) | |
| 22 | + else: | |
| 23 | + df[c] = df[c].map(lambda v: tex_escape("" if pd.isna(v) else v)) | |
| 24 | + cols = "l" + "r" * (len(df.columns) - 1) | |
| 25 | + head = " & ".join(f"\\textbf{{{tex_escape(c)}}}" for c in df.columns) + " \\\\" | |
| 26 | + body = "\n".join(" & ".join(map(str, r)) + " \\\\" for r in df.itertuples(index=False)) | |
| 27 | + env = "longtable" if longtable else "tabular" | |
| 28 | + s = f"% generated by tables/make_tables.py from {', '.join(sources)}\n" | |
| 29 | + if longtable: | |
| 30 | + s += f"\\begin{{longtable}}{{{cols}}}\n\\caption{{{caption}}}\\label{{{label}}}\\\\\n\\toprule\n{head}\n\\midrule\n\\endfirsthead\n\\toprule\n{head}\n\\midrule\n\\endhead\n{body}\n\\bottomrule\n" | |
| 31 | + if note: s += f"\\multicolumn{{{len(df.columns)}}}{{p{{0.95\\linewidth}}}}{{\\footnotesize {note}}}\\\\\n" | |
| 32 | + s += "\\end{longtable}\n" | |
| 33 | + else: | |
| 34 | + s += f"\\begin{{table}}[htbp]\\centering\\caption{{{caption}}}\\label{{{label}}}\n\\begin{{threeparttable}}\\small\n\\begin{{tabular}}{{{cols}}}\n\\toprule\n{head}\n\\midrule\n{body}\n\\bottomrule\n\\end{{tabular}}\n" | |
| 35 | + if note: s += f"\\begin{{tablenotes}}\\footnotesize\\item {note}\\end{{tablenotes}}\n" | |
| 36 | + s += "\\end{threeparttable}\\end{table}\n" | |
| 37 | + open(os.path.join(OUT, name + ".tex"), "w").write(s); INV.append({"file": name + ".tex", "caption": caption, "label": label, "sources": sources, "rows": int(len(df))}) | |
| 38 | + | |
| 39 | + | |
| 40 | +def main(): | |
| 41 | + p = f"{R}/data/metadata/prisma_counts.json" | |
| 42 | + if os.path.exists(p): | |
| 43 | + P = json.load(open(p)); rows = [("Identified: OpenAlex works with a CS topic, 2018–2026 (snapshot 2026-10-14, both passes)", P["identified"]["openalex_total_records_loaded"])] | |
| 44 | + labels = {"e1_outside_window": "E1 outside the window", "e2_type": "E2 non-research document type", "e3_xpac": "E3 XPAC expansion record (not DBLP-linked)", "e4_no_title": "E4 no title", "e5_duplicate": "E5 preprint merged into its published version", "e7_generic_deposit": "E7 generic deposit platform without DBLP/arXiv support"} | |
| 45 | + rows += [("Excluded " + labels.get(k, k), v) for k, v in P["excluded_sequential"].items()] | |
| 46 | + rows += [("After exclusions", P["after_exclusions"]), ("CS core (mapping corpus)", P["cs_core_eligible"]), ("CS penumbra (sensitivity set)", P["cs_penumbra_eligible_otherwise"]), ("Strict core: DBLP-indexed and primary field CS", P["strict_core_dblp_and_cs_primary"])] | |
| 47 | + write("tab_prisma", pd.DataFrame(rows, columns=["Stage", "Records"]), "Record flow of the mapping corpus (PRISMA-style, sequential exclusions).", "tab:prisma", ["data/metadata/prisma_counts.json"]) | |
| 48 | + write("tab_eligible_by_year", pd.DataFrame(sorted(P["eligible_by_year"].items()), columns=["Year", "Eligible works"]), "Eligible works by publication year (2026 incomplete).", "tab:eligible_year", ["data/metadata/prisma_counts.json"]) | |
| 49 | + write("tab_eligible_by_type", pd.DataFrame(P["eligible_by_type"].items(), columns=["OpenAlex type", "Eligible works"]), "Eligible works by document type.", "tab:eligible_type", ["data/metadata/prisma_counts.json"]) | |
| 50 | + p = f"{R}/data/metadata/linkage_report.json" | |
| 51 | + if os.path.exists(p): | |
| 52 | + L = json.load(open(p)); rows = [("DBLP records 2018–2026", L["dblp_2018_2026"]), ("… with DOI", L["dblp_with_doi"]), ("DBLP ↔ OpenAlex links by DOI", L["link_dblp_oa"].get("doi", 0)), ("… by normalised title + year (+ first author)", L["link_dblp_oa"].get("title_year", 0)), ("DBLP records linked (distinct)", L["dblp_linked_distinct"]), ("DBLP records without OpenAlex record", L["dblp_unlinked"]), | |
| 53 | + ("arXiv cs records created 2018–2026", L["arxiv_created_2018_2026"]), ("arXiv ↔ OpenAlex by arXiv id", L["link_arxiv_oa"].get("arxiv_id", 0)), ("… by DOI", L["link_arxiv_oa"].get("doi", 0)), ("… by title + year", L["link_arxiv_oa"].get("title_year", 0)), ("ACL entries 2018–2026", L["acl_2018_2026"]), ("ACL ↔ OpenAlex linked (2018–2026)", L["acl_linked_distinct_2018_2026"])] | |
| 54 | + write("tab_linkage", pd.DataFrame(rows, columns=["Quantity", "Count"]), "Record linkage between sources.", "tab:linkage", ["data/metadata/linkage_report.json"]) | |
| 55 | + for f, name, cap, cols in [("volume_yearly", "tab_volume_yearly", "Yearly volume and composition of the eligible corpus.", ["year", "n", "n_dblp", "n_cs_primary", "n_arxiv", "n_abstract", "n_oa", "n_preprint", "n_conf", "n_article", "mean_authors", "mean_refs"]), | |
| 56 | + ("citation_windows_yearly", "tab_citation_windows", "Fixed-window citation statistics by publication year (c12: publication year + 1; c36: + 2 calendar years).", ["year", "n", "mean_c12", "mean_c36", "med_c36", "share_uncited_36", "p99_c36", "gini_c36", "gini_c12", "mean_fwci"]), | |
| 57 | + ("team_size_yearly", "tab_team_size", "Team size and collaboration by year.", ["year", "n", "mean_authors", "med_authors", "single", "five_plus", "ten_plus", "intl", "multi_inst"]), | |
| 58 | + ("industry_share_yearly", "tab_industry", "Works with a company affiliation by year.", ["year", "n_with_inst", "n_company", "share_company"]), | |
| 59 | + ("venue_hhi_yearly", "tab_venue_hhi", "Venue concentration by year.", ["year", "n", "n_sources", "hhi", "top1_share"]), | |
| 60 | + ("diversity_yearly", "tab_diversity", "Rao–Stirling and Leinster–Cobbold diversity of the category mix by year (bootstrap 95\\% CI).", ["year", "n", "rao_stirling", "rs_ci_low", "rs_ci_high", "leinster_cobbold_q2"]), | |
| 61 | + ("cagr_by_category", "tab_cagr", "Compound annual growth by category, baseline (2018–2022) and post (2023–2025).", ["category", "n_2018", "n_2022", "n_2025", "cagr_baseline", "cagr_post", "share_2018", "share_2025"])]: | |
| 62 | + p = f"{R}/results/bibliometrics/{f}.csv" | |
| 63 | + if os.path.exists(p): | |
| 64 | + d = pd.read_csv(p); d = d[[c for c in cols if c in d.columns]] | |
| 65 | + if "team" in f or "volume" in f: d = d[d.iloc[:, 0] <= 2026] | |
| 66 | + write(name, d, cap, f"tab:{name[4:]}", [f"results/bibliometrics/{f}.csv"], longtable=len(d) > 30) | |
| 67 | + p = f"{R}/results/bibliometrics/countries_yearly.csv" | |
| 68 | + if os.path.exists(p): | |
| 69 | + d = pd.read_csv(p); top = d.groupby("country").n_frac.sum().nlargest(15).index; piv = d[d.country.isin(top)].pivot_table(index="country", columns="year", values="n_frac").loc[top] | |
| 70 | + piv = piv.round(0).astype(int).reset_index(); write("tab_countries", piv, "Fractional output of the fifteen largest countries by year.", "tab:countries", [p]) | |
| 71 | + p = f"{R}/results/econometrics/breaks.csv" | |
| 72 | + if os.path.exists(p): | |
| 73 | + d = pd.read_csv(p); cols = ["series", "chatgpt_level", "chatgpt_level_p", "chatgpt_slope", "chatgpt_slope_p", "placebo_level_p", "placebo_slope_p", "bp_breaks_bic", "pelt"] | |
| 74 | + write("tab_breaks", d[[c for c in cols if c in d.columns]], "Structural-change estimates by series: ITS level and slope changes at 2022-12 (HAC p-values), placebo p-values, Bai–Perron (BIC) and PELT break dates.", "tab:breaks", [p], longtable=len(d) > 25) | |
| 75 | + p = f"{R}/taxonomy/validation/metrics.json" | |
| 76 | + if os.path.exists(p): | |
| 77 | + M = json.load(open(p)); rows = [] | |
| 78 | + for k, v in M.items(): | |
| 79 | + if isinstance(v, dict) and "per_category" in v: | |
| 80 | + for c, m in v["per_category"].items(): | |
| 81 | + rows.append({"comparison": k, "category": c, "precision": m["precision"], "recall": m["recall"], "f1": m["f1"], "tp": m["tp"], "fp": m["fp"], "fn": m["fn"]}) | |
| 82 | + if rows: write("tab_validation", pd.DataFrame(rows), "Taxonomy validation: precision, recall and F1 of the rule-based assignment against each rater and the rater consensus, by category.", "tab:validation", [p], longtable=True) | |
| 83 | + for f in glob.glob(f"{R}/results/emerging/metrics_h*.json"): | |
| 84 | + M = json.load(open(f)); rows = [{"model": m, "AUC": M[m]["auc"], "P@20": M[m]["precision_at_20"], "P@50": M[m]["precision_at_50"]} for m in ("zscore_baseline", "logistic", "gbm") if m in M] | |
| 85 | + write("tab_emerging_h%d" % M["horizon"], pd.DataFrame(rows), f"Out-of-time performance of the emerging-topic detectors at horizon {M['horizon']} months (test cutoffs {M['test_cutoffs'][0]}–{M['test_cutoffs'][-1]}).", f"tab:emerging_h{M['horizon']}", [f]) | |
| 86 | + p = f"{R}/results/emerging/known_set_lead_times.csv" | |
| 87 | + if os.path.exists(p): | |
| 88 | + d = pd.read_csv(p)[["concept", "total_works", "detection_month", "peak_growth_month", "lead_months"]]; write("tab_known_set", d, "Preregistered known-emergence set: detection month, peak-growth month and lead time.", "tab:known_set", [p]) | |
| 89 | + json.dump(INV, open(os.path.join(OUT, "TABLES.json"), "w"), indent=1); print(len(INV), "tables") | |
| 90 | + | |
| 91 | + | |
| 92 | +if __name__ == "__main__": | |
| 93 | + main() | |
| 94 | ||