WP44: generators for Appendix F (supplementary tables) and G (figure documentation)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
5 changed files +73 −58
modified
manuscript/appendices/appF_supplementary_tables.tex
+19 −1
@@ -1 +1,19 @@ | ||
| 1 | −% appF_supplementary_tables — to be written (Phase 7) | |
| 1 | +\section{Supplementary tables}\label{app:supp-tables} | |
| 2 | +This appendix collects the generated tables not shown in the main text. Every table is produced by \texttt{tables/make\_tables.py} from the result file(s) named in its first comment line; none is edited by hand. | |
| 3 | + | |
| 4 | +\input{tables/tab_prisma} | |
| 5 | +\clearpage | |
| 6 | +\input{tables/tab_eligible_by_year} | |
| 7 | +\clearpage | |
| 8 | +\input{tables/tab_eligible_by_type} | |
| 9 | +\clearpage | |
| 10 | +\input{tables/tab_linkage} | |
| 11 | +\clearpage | |
| 12 | +\input{tables/tab_volume_yearly} | |
| 13 | +\clearpage | |
| 14 | +\input{tables/tab_team_size} | |
| 15 | +\clearpage | |
| 16 | +\input{tables/tab_breaks} | |
| 17 | +\clearpage | |
| 18 | +\input{tables/tab_known_set} | |
| 19 | +\clearpage | |
modified
manuscript/appendices/appG_figure_documentation.tex
+21 −1
@@ -1 +1,21 @@ | ||
| 1 | −% appG_figure_documentation — to be written (Phase 7) | |
| 1 | +\section{Figure documentation}\label{app:figures} | |
| 2 | +All figures are generated by \texttt{figures/make\_figures.py} (matplotlib, vector PDF) with the validated palette of \texttt{figures/style.py}; the table lists, for each figure, its caption and the result artefacts it reads. Colours follow a fixed categorical order; one axis per panel; direct labels for at most four series; the 2022-11-30 reference date is drawn as a dashed line and never as a series. | |
| 3 | + | |
| 4 | +\begin{longtable}{p{0.22\linewidth}p{0.46\linewidth}p{0.26\linewidth}} | |
| 5 | +\caption{Figure inventory.}\label{tab:figure-inventory}\\ | |
| 6 | +\toprule \textbf{File} & \textbf{Caption} & \textbf{Sources} \\ \midrule \endfirsthead | |
| 7 | +\toprule \textbf{File} & \textbf{Caption} & \textbf{Sources} \\ \midrule \endhead | |
| 8 | +\texttt{fig\_prisma\_flow.pdf} & PRISMA-style flow of record identification, exclusion and the final mapping corpus. & prisma\_counts.json \\ | |
| 9 | +\texttt{fig\_volume\_yearly.pdf} & Eligible works per publication year by source membership (2026 incomplete). & volume\_yearly.csv \\ | |
| 10 | +\texttt{fig\_volume\_monthly\_raw.pdf} & Monthly first appearances; January spikes reflect year-only publication dates and motivate the month-precision filter. & volume\_monthly.csv \\ | |
| 11 | +\texttt{fig\_types\_openness.pdf} & Document-type composition and metadata completeness by year. & volume\_yearly.csv \\ | |
| 12 | +\texttt{fig\_collaboration.pdf} & Team size, international co-authorship and industry participation by year. & team\_size\_yearly.csv; industry\_share\_yearly.csv \\ | |
| 13 | +\texttt{fig\_citation\_inequality.pdf} & Fixed-window citation levels and inequality by publication year (years with complete 36-month windows). & citation\_windows\_yearly.csv \\ | |
| 14 | +\texttt{fig\_field\_inflows.pdf} & Share of references pointing outside the corpus, by cited OpenAlex field and citing year. & field\_flows\_yearly.csv \\ | |
| 15 | +\texttt{fig\_cited\_age.pdf} & Distribution of the age of cited works for selected citing years. & cited\_age\_yearly.csv \\ | |
| 16 | +\texttt{fig\_venue\_concentration.pdf} & Number of distinct venues and venue concentration (HHI, top-source share) by year. & venue\_hhi\_yearly.csv \\ | |
| 17 | +\texttt{fig\_country\_shares.pdf} & Country shares of output with fractional counting, top eight countries. & countries\_yearly.csv \\ | |
| 18 | +\texttt{fig\_its\_total.pdf} & Monthly output with the interrupted-time-series fit; level and slope changes with HAC and placebo p-values. & fit\_total.csv; its\_results.json \\ | |
| 19 | +\texttt{fig\_placebo\_total.pdf} & Placebo distribution of ITS t-statistics over every candidate event month; the true event is highlighted. & placebo\_total.csv \\ | |
| 20 | +\bottomrule | |
| 21 | +\end{longtable} | |
modified
manuscript/references_part03.bib
+0 −48
@@ -4,16 +4,6 @@ | ||
| 4 | 4 | % alperin2024openalex, ginsparg2011arxiv, haddaway2022prismaflow, hendricks2020crossref, ley2002dblp, ley2009dblp, page2021prisma, petersen2008mapping, petersen2015mapping, priem2022openalex, rethlefsen2021prismas, tricco2018prismascr, visser2021comparison, waltman2016review, wang2020mag |
| 5 | 5 | % (their verification rows remain in references_verification_part03.csv with note=reused:references.bib). |
| 6 | 6 | |
| 7 | −@article{page2021prismaee, | |
| 8 | − author = {Page, Matthew J. and Moher, David and Bossuyt, Patrick M. and Boutron, Isabelle and Hoffmann, Tammy C. and Mulrow, Cynthia D. and Shamseer, Larissa and Tetzlaff, Jennifer M. and Akl, Elie A. and Brennan, Sue E. and Chou, Roger and Glanville, Julie and Grimshaw, Jeremy M. and Hr{\'o}bjartsson, Asbj{\o}rn and Lalu, Manoj M. and Li, Tianjing and Loder, Elizabeth W. and Mayo-Wilson, Evan and McDonald, Steve and McGuinness, Luke A. and Stewart, Lesley A. and Thomas, James and Tricco, Andrea C. and Welch, Vivian A. and Whiting, Penny and McKenzie, Joanne E.}, | |
| 9 | − title = {{PRISMA} 2020 explanation and elaboration: updated guidance and exemplars for reporting systematic reviews}, | |
| 10 | − journal = {BMJ}, | |
| 11 | − year = {2021}, | |
| 12 | − volume = {372}, | |
| 13 | − pages = {n160}, | |
| 14 | − doi = {10.1136/bmj.n160}, | |
| 15 | − note = {verified: crossref 2026-11-03} | |
| 16 | −} | |
| 17 | 7 | @article{gusenbauer2020search, |
| 18 | 8 | author = {Gusenbauer, Michael and Haddaway, Neal R.}, |
| 19 | 9 | title = {Which academic search systems are suitable for systematic reviews or meta-analyses? {Evaluating} retrieval qualities of {Google Scholar}, {PubMed}, and 26 other resources}, |
@@ -34,24 +24,6 @@ | ||
| 34 | 24 | howpublished = {arXiv preprint arXiv:1905.00075}, |
| 35 | 25 | note = {verified: arxiv 2026-11-03} |
| 36 | 26 | } |
| 37 | −@article{culbert2025openalex, | |
| 38 | − author = {Culbert, Jack H. and Hobert, Anne and Jahn, Najko and Haupka, Nick and Schmidt, Marion and Donner, Paul and Mayr, Philipp}, | |
| 39 | − title = {Reference coverage analysis of {OpenAlex} compared to {Web of Science} and {Scopus}}, | |
| 40 | − journal = {Scientometrics}, | |
| 41 | − year = {2025}, | |
| 42 | − volume = {130}, | |
| 43 | − pages = {2475--2492}, | |
| 44 | − doi = {10.1007/s11192-025-05293-3}, | |
| 45 | − note = {verified: crossref 2026-11-03} | |
| 46 | −} | |
| 47 | −@article{scheidsteger2023openalex, | |
| 48 | − author = {Scheidsteger, Thomas and Haunschild, Robin}, | |
| 49 | − title = {Which of the metadata with relevance for bibliometrics are the same and which are different when switching from {Microsoft Academic Graph} to {OpenAlex}?}, | |
| 50 | − journal = {Profesional de la informaci{\'o}n}, | |
| 51 | − year = {2023}, | |
| 52 | − doi = {10.3145/epi.2023.mar.09}, | |
| 53 | − note = {verified: crossref 2026-11-03} | |
| 54 | −} | |
| 55 | 27 | @article{delgadoquiros2024completeness, |
| 56 | 28 | author = {Delgado-Quir{\'o}s, Lorena and Ortega, Jos{\'e} Luis}, |
| 57 | 29 | title = {Completeness degree of publication metadata in eight free-access scholarly databases}, |
@@ -93,16 +65,6 @@ | ||
| 93 | 65 | doi = {10.1007/978-3-642-31164-2}, |
| 94 | 66 | note = {verified: crossref 2026-11-03} |
| 95 | 67 | } |
| 96 | −@article{lin2020preprints, | |
| 97 | − author = {Lin, Jialiang and Yu, Yao and Zhou, Yu and Zhou, Zhiyang and Shi, Xiaodong}, | |
| 98 | − title = {How many preprints have actually been printed and why: a case study of computer science preprints on {arXiv}}, | |
| 99 | − journal = {Scientometrics}, | |
| 100 | − year = {2020}, | |
| 101 | − volume = {124}, | |
| 102 | − pages = {555--574}, | |
| 103 | − doi = {10.1007/s11192-020-03430-8}, | |
| 104 | − note = {verified: crossref 2026-11-03} | |
| 105 | −} | |
| 106 | 68 | @article{cabanac2021preprint, |
| 107 | 69 | author = {Cabanac, Guillaume and Oikonomidi, Theodora and Boutron, Isabelle}, |
| 108 | 70 | title = {Day-to-day discovery of preprint--publication links}, |
@@ -113,16 +75,6 @@ | ||
| 113 | 75 | doi = {10.1007/s11192-021-03900-7}, |
| 114 | 76 | note = {verified: crossref 2026-11-03} |
| 115 | 77 | } |
| 116 | −@article{wang2013window, | |
| 117 | − author = {Wang, Jian}, | |
| 118 | − title = {Citation time window choice for research impact evaluation}, | |
| 119 | − journal = {Scientometrics}, | |
| 120 | − year = {2013}, | |
| 121 | − volume = {94}, | |
| 122 | − pages = {851--872}, | |
| 123 | − doi = {10.1007/s11192-012-0775-9}, | |
| 124 | − note = {verified: crossref 2026-11-03 (online 2012, volume 94 = 2013)} | |
| 125 | −} | |
| 126 | 78 | @article{nosek2018preregistration, |
| 127 | 79 | author = {Nosek, Brian A. and Ebersole, Charles R. and DeHaven, Alexander C. and Mellor, David T.}, |
| 128 | 80 | title = {The preregistration revolution}, |
modified
manuscript/sections/part03_data.tex
+8 −8
@@ -16,7 +16,7 @@ these choices for WP44 and reports, stage by stage, how the mapping corpus was a | ||
| 16 | 16 | sources. The exposition follows the order of the pipeline (identification, linkage, deduplication, |
| 17 | 17 | eligibility, scope, dating) and closes with the quality indicators and the research-integrity procedures that |
| 18 | 18 | govern every later analysis. The reporting structure is modelled on the PRISMA 2020 items that apply to a |
| 19 | −mapping study \citep{page2021prisma,page2021prismaee}; items concerned with risk of bias and effect measures | |
| 19 | +mapping study \citep{page2021prisma,page2021prismaexplanation}; items concerned with risk of bias and effect measures | |
| 20 | 20 | have no counterpart in a bibliometric design and are replaced, as PRISMA-ScR does for scoping reviews |
| 21 | 21 | \citep{tricco2018prismascr}, by indicators of metadata quality and coverage (Section~\ref{sec:data-quality}). |
| 22 | 22 | The search-reporting recommendations of PRISMA-S \citep{rethlefsen2021prismas} are followed to the extent |
@@ -33,8 +33,8 @@ coverage between Web of Science, Scopus, Dimensions, Crossref and Microsoft Acad | ||
| 33 | 33 | indexes is known to under-represent conference proceedings, which carry a large share of computer-science |
| 34 | 34 | output \citep{mongeon2016coverage}. For the open successor of Microsoft Academic, OpenAlex, the first |
| 35 | 35 | independent assessments report strong coverage of works with DOIs together with weaker and uneven metadata on |
| 36 | −affiliations, document types and references \citep{alperin2024openalex,culbert2025openalex, | |
| 37 | −scheidsteger2023openalex,delgadoquiros2024completeness}. These findings motivate the design adopted here: one | |
| 36 | +affiliations, document types and references \citep{alperin2024openalex,culbert2025openalexcoverage, | |
| 37 | +scheidsteger2023magopenalex,delgadoquiros2024completeness}. These findings motivate the design adopted here: one | |
| 38 | 38 | source supplies the backbone (OpenAlex), a second supplies the definition of what counts as computer science |
| 39 | 39 | (DBLP), a third supplies the preprint layer and its dates (arXiv), and a fourth checks the coverage of a |
| 40 | 40 | subfield with a dense, curated venue structure (the ACL Anthology). Crossref is used for verification only. |
@@ -278,7 +278,7 @@ for the linkage-precision review. | ||
| 278 | 278 | For arXiv, the order is: (i) arXiv identifier equality against the identifier OpenAlex records for the work, |
| 279 | 279 | (ii) DOI equality for e-prints that carry a journal DOI, (iii) normalised title equality with a year |
| 280 | 280 | difference of at most two, the wider tolerance reflecting the typical lag between a preprint and its published |
| 281 | −version \citep{lin2020preprints}. The steps linked \LinkLinkArxivOaArxivId{}, \LinkLinkArxivOaDoi{} and | |
| 281 | +version \citep{lin2020arxivcs}. The steps linked \LinkLinkArxivOaArxivId{}, \LinkLinkArxivOaDoi{} and | |
| 282 | 282 | \LinkLinkArxivOaTitleYear{} records respectively, for \LinkArxivLinkedDistinct{} distinct e-prints linked. |
| 283 | 283 | For the ACL Anthology, the order is: (i) DOI equality, (ii) equality between the Anthology identifier and the |
| 284 | 284 | identifier embedded in an \texttt{aclanthology.org} landing-page URL among the work's OpenAlex locations, |
@@ -400,7 +400,7 @@ both is counted once, under either route). Set against the \PrismaEligibleByType | ||
| 400 | 400 | published counterpart, the figure shows that most e-prints in the candidate set were not matched to a |
| 401 | 401 | published version within OpenAlex. Three explanations coexist and cannot be separated without manual review: |
| 402 | 402 | many e-prints are genuinely unpublished, consistent with the publication rates reported for computer-science |
| 403 | −preprints \citep{lin2020preprints}; some published versions carry a modified title; and some published | |
| 403 | +preprints \citep{lin2020arxivcs}; some published versions carry a modified title; and some published | |
| 404 | 404 | versions are not in OpenAlex or lack a computer-science topic. The first explanation is benign; the second and |
| 405 | 405 | third leave a residual double count whose size is bounded in Part~IX by re-running the pairing with a title |
| 406 | 406 | similarity threshold rather than exact equality and reporting the difference as a sensitivity, not as a |
@@ -697,7 +697,7 @@ published but not yet indexed (indexing lag), works indexed but not yet linked t | ||
| 697 | 697 | own curation lags the publishers, and works whose citation counts have had little time to accumulate. The |
| 698 | 698 | first two shrink the counts of the recent months; the third affects only impact indicators and is treated in |
| 699 | 699 | Part~IV through fixed citation windows, which are the standard remedy for the age dependence of citation |
| 700 | −counts \citep{wang2013window,waltman2016review}. | |
| 700 | +counts \citep{wang2013citationwindow,waltman2016review}. | |
| 701 | 701 | |
| 702 | 702 | For the counts, the diagnosis uses the sources against one another. The DBLP dump of 2026-11-02 holds, for |
| 703 | 703 | 2026, a count well below its 2025 count although ten months of the year have elapsed, which shows that DBLP's |
@@ -740,7 +740,7 @@ reference list, presence of at least one affiliation with an institution identif | ||
| 740 | 740 | country for at least one author. In the eligible corpus, \PrismaEligibleMembershipHasAbstract{} works have |
| 741 | 741 | an abstract and \PrismaEligibleMembershipHasReferences{} have at least one resolved reference, out of |
| 742 | 742 | \PrismaCsCoreEligible{}. Both shares are lower than those reported for the DOI-bearing journal literature in |
| 743 | −comparisons of OpenAlex with the commercial indexes \citep{culbert2025openalex,delgadoquiros2024completeness}, | |
| 743 | +comparisons of OpenAlex with the commercial indexes \citep{culbert2025openalexcoverage,delgadoquiros2024completeness}, | |
| 744 | 744 | which is expected in a corpus that includes dissertations, book chapters and preprints, and both fall sharply |
| 745 | 745 | for the most recent year (Figure~\ref{fig:types-openness}). The consequences are concrete. A work without an |
| 746 | 746 | abstract cannot be embedded or classified by its content and is classified from its venue, its arXiv |
@@ -840,7 +840,7 @@ None of these biases is corrected by reweighting. The study reports them, bounds | ||
| 840 | 840 | permits, and re-estimates its results on alternative corpora (strict core; corpus with penumbra; corpus with |
| 841 | 841 | XPAC research types; corpus with DBLP-only records; corpus with generic deposits) so that the reader can see |
| 842 | 842 | how far each conclusion depends on each scope decision. This is the approach that the science-of-science |
| 843 | −literature recommends for databases whose coverage is known to be uneven \citep{fortunato2018sos}, and it | |
| 843 | +literature recommends for databases whose coverage is known to be uneven \citep{fortunato2018scienceofscience}, and it | |
| 844 | 844 | is the only one compatible with a mandate that forbids the fabrication of corrections. |
| 845 | 845 | |
| 846 | 846 | \section{Research-integrity procedures} |
added
tables/make_appendix_fg.py
+25 −0
@@ -0,0 +1,25 @@ | ||
| 1 | +#!/usr/bin/env python3 | |
| 2 | +"""Generate Appendix F (supplementary tables: every generated table not \\input elsewhere in the manuscript) and | |
| 3 | +Appendix G (figure documentation: file, caption, generating script, source artefacts) from manuscript/tables/TABLES.json and | |
| 4 | +figures/FIGURES.json. Run after tables/make_tables.py and figures/make_figures.py.""" | |
| 5 | +import json, os, re, glob, sys | |
| 6 | +R = sys.argv[1] if len(sys.argv) > 1 else os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| 7 | +M = os.path.join(R, "manuscript") | |
| 8 | +used = set() | |
| 9 | +for f in glob.glob(f"{M}/sections/*.tex") + [p for p in glob.glob(f"{M}/appendices/*.tex") if not p.endswith("appF_supplementary_tables.tex")]: | |
| 10 | + used |= set(re.findall(r"\\input\{tables/([A-Za-z0-9_]+)\}", open(f).read())) | |
| 11 | +tables = json.load(open(f"{M}/tables/TABLES.json")) | |
| 12 | +rest = [t for t in tables if t["file"].replace(".tex", "") not in used] | |
| 13 | +esc = lambda s: str(s).replace("_", "\\_").replace("&", "\\&").replace("%", "\\%") | |
| 14 | +out = ["\\section{Supplementary tables}\\label{app:supp-tables}", "This appendix collects the generated tables not shown in the main text. Every table is produced by \\texttt{tables/make\\_tables.py} from the result file(s) named in its first comment line; none is edited by hand.", ""] | |
| 15 | +for t in rest: | |
| 16 | + out.append(f"\\input{{tables/{t['file'].replace('.tex','')}}}\n\\clearpage") | |
| 17 | +open(f"{M}/appendices/appF_supplementary_tables.tex", "w").write("\n".join(out) + "\n") | |
| 18 | +figs = json.load(open(f"{R}/figures/FIGURES.json")) | |
| 19 | +g = ["\\section{Figure documentation}\\label{app:figures}", "All figures are generated by \\texttt{figures/make\\_figures.py} (matplotlib, vector PDF) with the validated palette of \\texttt{figures/style.py}; the table lists, for each figure, its caption and the result artefacts it reads. Colours follow a fixed categorical order; one axis per panel; direct labels for at most four series; the 2022-11-30 reference date is drawn as a dashed line and never as a series.", "", | |
| 20 | + "\\begin{longtable}{p{0.22\\linewidth}p{0.46\\linewidth}p{0.26\\linewidth}}", "\\caption{Figure inventory.}\\label{tab:figure-inventory}\\\\", "\\toprule \\textbf{File} & \\textbf{Caption} & \\textbf{Sources} \\\\ \\midrule \\endfirsthead", "\\toprule \\textbf{File} & \\textbf{Caption} & \\textbf{Sources} \\\\ \\midrule \\endhead"] | |
| 21 | +for f in figs: | |
| 22 | + g.append(f"\\texttt{{{esc(f['file'])}.pdf}} & {esc(f['caption'])} & {esc('; '.join(os.path.basename(s) for s in f['sources']))} \\\\") | |
| 23 | +g += ["\\bottomrule", "\\end{longtable}"] | |
| 24 | +open(f"{M}/appendices/appG_figure_documentation.tex", "w").write("\n".join(g) + "\n") | |
| 25 | +print(len(rest), "supplementary tables;", len(figs), "figures documented") | |
| 26 | ||