SPB Git
spboucher.ai

spb/wp44_uqo

Public

WP44: corpus v2 metadata (7.11 M core works), configurable DuckDB memory

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Nov 3, 2026) parent f781212 Browse files

13 changed files +263 −195

13 changed files
Unified Split
modified bibliometrics/02_collaboration.py +1 −1
@@ -10,7 +10,7 @@ import argparse, os, duckdb
10 def main(): 10 def main():
11 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); a = ap.parse_args(); R = a.root 11 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); a = ap.parse_args(); R = a.root
12 out = os.path.join(R, "results/bibliometrics"); os.makedirs(out, exist_ok=True) 12 out = os.path.join(R, "results/bibliometrics"); os.makedirs(out, exist_ok=True)
13− con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb"), read_only=True); con.execute("SET threads TO 20; SET memory_limit='60GB';") 13+ con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb"), read_only=True); con.execute("SET threads TO 20; SET memory_limit='" + os.environ.get("WP44_DUCKDB_MEM", "60GB") + "';")
14 con.execute("CREATE OR REPLACE TEMP VIEW E AS SELECT * FROM corpus_works WHERE eligible") 14 con.execute("CREATE OR REPLACE TEMP VIEW E AS SELECT * FROM corpus_works WHERE eligible")
15 con.execute(f"""COPY (SELECT publication_year AS year, period, count(*) n, round(avg(n_authors),3) mean_authors, quantile_cont(n_authors,0.5) med_authors, 15 con.execute(f"""COPY (SELECT publication_year AS year, period, count(*) n, round(avg(n_authors),3) mean_authors, quantile_cont(n_authors,0.5) med_authors,
16 sum((n_authors=1)::int) single, sum((n_authors>=5)::int) five_plus, sum((n_authors>=10)::int) ten_plus, sum((n_countries>=2)::int) intl, 16 sum((n_authors=1)::int) single, sum((n_authors>=5)::int) five_plus, sum((n_authors>=10)::int) ten_plus, sum((n_countries>=2)::int) intl,
modified bibliometrics/03_citation_impact.py +1 −1
@@ -14,7 +14,7 @@ import argparse, os, duckdb
14 def main(): 14 def main():
15 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); a = ap.parse_args(); R = a.root 15 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); a = ap.parse_args(); R = a.root
16 out = os.path.join(R, "results/bibliometrics"); os.makedirs(out, exist_ok=True) 16 out = os.path.join(R, "results/bibliometrics"); os.makedirs(out, exist_ok=True)
17− con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb")); con.execute("SET threads TO 20; SET memory_limit='60GB';") 17+ con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb")); con.execute("SET threads TO 20; SET memory_limit='" + os.environ.get("WP44_DUCKDB_MEM", "60GB") + "';")
18 con.execute("""CREATE OR REPLACE TABLE work_citation_windows AS 18 con.execute("""CREATE OR REPLACE TABLE work_citation_windows AS
19 WITH E AS (SELECT id, preprint_work_id, publication_year AS y, coalesce(type, 'unknown') AS type, coalesce(primary_subfield_id, -1) AS sf, cited_by_count, fwci 19 WITH E AS (SELECT id, preprint_work_id, publication_year AS y, coalesce(type, 'unknown') AS type, coalesce(primary_subfield_id, -1) AS sf, cited_by_count, fwci
20 FROM corpus_works WHERE eligible AND NOT e6_retracted), 20 FROM corpus_works WHERE eligible AND NOT e6_retracted),
added data/metadata/dblp_only_counts.csv +54 −0
@@ -0,0 +1,54 @@
1+year,rec_type,n,with_doi
2+2018,article,22769,19614
3+2018,book,89,4
4+2018,data,246,245
5+2018,incollection,171,115
6+2018,inproceedings,11322,508
7+2018,phdthesis,4430,43
8+2019,article,29350,26379
9+2019,book,115,4
10+2019,data,471,444
11+2019,incollection,30,5
12+2019,inproceedings,11656,546
13+2019,phdthesis,4162,35
14+2020,article,37639,34805
15+2020,book,62,5
16+2020,data,816,807
17+2020,incollection,52,5
18+2020,inproceedings,13639,610
19+2020,phdthesis,3857,79
20+2021,article,39300,36294
21+2021,book,80,5
22+2021,data,972,927
23+2021,incollection,47,0
24+2021,inproceedings,15423,588
25+2021,phdthesis,4222,86
26+2022,article,46967,43149
27+2022,book,70,1
28+2022,data,1335,1265
29+2022,incollection,33,2
30+2022,inproceedings,19180,574
31+2022,phdthesis,4133,138
32+2023,article,56159,51893
33+2023,book,88,1
34+2023,data,1773,1691
35+2023,incollection,21,21
36+2023,inproceedings,18807,827
37+2023,phdthesis,3648,97
38+2024,article,66262,61366
39+2024,book,60,2
40+2024,data,2471,2372
41+2024,incollection,23,0
42+2024,inproceedings,24391,646
43+2024,phdthesis,3343,69
44+2025,article,53124,47735
45+2025,book,44,0
46+2025,data,3835,3782
47+2025,incollection,41,0
48+2025,inproceedings,26050,692
49+2025,phdthesis,2099,1
50+2026,article,16280,11901
51+2026,book,21,2
52+2026,data,2765,2741
53+2026,inproceedings,18840,3473
54+2026,phdthesis,398,0
modified data/metadata/linkage_report.json +23 −24
@@ -1,45 +1,44 @@
1 { 1 {
2− "created": "2026-11-03T20:50:56.182276+00:00", 2+ "created": "2026-11-03T23:03:10.160666+00:00",
3− "oa_cs_records": 15490159, 3+ "oa_cs_records": 16592966,
4− "oa_cs_distinct_ids": 15490159, 4+ "oa_cs_distinct_ids": 16592966,
5 "dblp_2018_2026": 4347365, 5 "dblp_2018_2026": 4347365,
6 "dblp_with_doi": 4052326, 6 "dblp_with_doi": 4052326,
7 "dblp_with_arxiv": 0, 7 "dblp_with_arxiv": 0,
8 "link_dblp_oa": { 8 "link_dblp_oa": {
9− "title_year": 179990, 9+ "title_year": 141062,
10− "doi": 2562431 10+ "doi": 3639848
11 }, 11 },
12− "dblp_linked_distinct": 2736903, 12+ "dblp_linked_distinct": 3774184,
13− "dblp_unlinked": 1610462, 13+ "dblp_unlinked": 573181,
14 "dblp_unlinked_by_type": { 14 "dblp_unlinked_by_type": {
15− "phdthesis": 26841, 15+ "incollection": 418,
16− "incollection": 3362, 16+ "book": 629,
17− "inproceedings": 477235, 17+ "phdthesis": 30292,
18− "book": 1398, 18+ "inproceedings": 159308,
19− "article": 1084640, 19+ "article": 367850,
20− "data": 16986 20+ "data": 14684
21 }, 21 },
22− "dblp_unlinked_with_doi_in_slim": 1077407, 22+ "dblp_unlinked_with_doi_in_slim": 1605,
23 "arxiv_cs_records": 915115, 23 "arxiv_cs_records": 915115,
24 "arxiv_created_2018_2026": 890852, 24 "arxiv_created_2018_2026": 890852,
25 "link_arxiv_oa": { 25 "link_arxiv_oa": {
26− "arxiv_id": 792672, 26+ "title_year": 1768,
27− "doi": 520, 27+ "doi": 783,
28− "title_year": 2143 28+ "arxiv_id": 914406
29 }, 29 },
30− "arxiv_linked_distinct": 744680, 30+ "arxiv_linked_distinct": 864677,
31 "acl_records": 135558, 31 "acl_records": 135558,
32 "acl_2018_2026": 85625, 32 "acl_2018_2026": 85625,
33 "link_acl_oa": { 33 "link_acl_oa": {
34− "doi": 57474, 34+ "doi": 58174,
35− "title_year": 14904 35+ "title_year": 15004
36 }, 36 },
37− "acl_linked_distinct_2018_2026": 72223, 37+ "acl_linked_distinct_2018_2026": 72949,
38 "needed_oa_ids_second_pass": { 38 "needed_oa_ids_second_pass": {
39− "dblp_doi": 1077406, 39+ "dblp_doi": 1605
40− "arxiv_doi": 39200
41 }, 40 },
42 "slim_total": 464765485, 41 "slim_total": 464765485,
43 "slim_cs_any_2018_2026": 15490159, 42 "slim_cs_any_2018_2026": 15490159,
44− "seconds": 10 43+ "seconds": 12
45 } 44 }
\ No newline at end of file
modified data/metadata/prisma_counts.json +79 −69
@@ -1,98 +1,108 @@
1 { 1 {
2− "generated": "2026-11-03T21:08:38.910490+00:00", 2+ "generated": "2026-11-03T23:03:34.908225+00:00",
3 "window": [ 3 "window": [
4 "2018-01-01", 4 "2018-01-01",
5 "2026-11-02" 5 "2026-11-02"
6 ], 6 ],
7 "boundary": "2022-11-30", 7 "boundary": "2022-11-30",
8 "identified": { 8 "identified": {
9− "openalex_cs_topic_2018_2026_pass1": 15490159, 9+ "openalex_cs_topic_2018_2026_pass1": 16592966,
10− "openalex_total_records_loaded": 15490159, 10+ "openalex_total_records_loaded": 16592966,
11 "dblp_2018_2026": 4347365, 11 "dblp_2018_2026": 4347365,
12 "arxiv_cs_2018_2026": 890852, 12 "arxiv_cs_2018_2026": 890852,
13 "acl_2018_2026": 85625 13 "acl_2018_2026": 85625
14 }, 14 },
15 "linked": { 15 "linked": {
16− "dblp_to_openalex": 2736903, 16+ "dblp_to_openalex": 3774184,
17− "arxiv_to_openalex": 744680, 17+ "arxiv_to_openalex": 864677,
18− "acl_to_openalex": 72256, 18+ "acl_to_openalex": 73054,
19− "dblp_only_no_openalex_record": 1610462 19+ "dblp_only_no_openalex_record": 573181
20 }, 20 },
21 "excluded": { 21 "excluded": {
22− "e1_outside_window": 160, 22+ "e1_outside_window": 59453,
23− "e2_type": 3961633, 23+ "e2_type": 3995235,
24− "e3_xpac": 3950876, 24+ "e3_xpac": 3974822,
25− "e4_no_title": 16426, 25+ "e4_no_title": 16442,
26− "e5_duplicate": 114115, 26+ "e5_duplicate": 120147,
27− "e6_retracted": 25165, 27+ "e7_generic_deposit": 1277584,
28− "xpac_kept_dblp": 87288 28+ "e6_retracted": 28587,
29+ "xpac_kept_dblp": 74771
29 }, 30 },
30 "excluded_e2_by_type": { 31 "excluded_e2_by_type": {
31− "dataset": 2775759, 32+ "dataset": 2779058,
32− "other": 522654, 33+ "other": 538613,
33 "libguides": 191658, 34 "libguides": 191658,
34− "software": 177500, 35+ "software": 177515,
35− "conference-abstract": 66267, 36+ "conference-abstract": 68101,
36− "peer-review": 60176, 37+ "peer-review": 60180,
37− "paratext": 39162, 38+ "paratext": 41468,
38− "reference-entry": 32247, 39+ "reference-entry": 33290,
39− "editorial": 24907, 40+ "editorial": 29543,
40− "supplementary-materials": 18882, 41+ "erratum": 20850,
41− "erratum": 17340, 42+ "supplementary-materials": 18946,
42− "standard": 10113, 43+ "standard": 10125,
43− "book-review": 9517, 44+ "book-review": 9524,
44− "retraction": 8217, 45+ "retraction": 8924,
45− "data-paper": 5255, 46+ "data-paper": 5308,
46− "letter": 1979 47+ "letter": 2132
47 }, 48 },
48 "excluded_sequential": { 49 "excluded_sequential": {
49− "e1_outside_window": 160, 50+ "e1_outside_window": 59453,
50− "e2_type": 3961589, 51+ "e2_type": 3994472,
51− "e3_xpac": 1421692, 52+ "e3_xpac": 1430851,
52− "e4_no_title": 1188, 53+ "e4_no_title": 1193,
53− "e5_duplicate": 106928 54+ "e5_duplicate": 114421,
55+ "e7_generic_deposit": 473549
54 }, 56 },
55− "after_exclusions": 9998602, 57+ "after_exclusions": 10519027,
56− "cs_core_eligible": 6403238, 58+ "version_relationship": {
57− "cs_penumbra_eligible_otherwise": 3595364, 59+ "arxiv_id": 13175,
60+ "title_year": 84776
61+ },
62+ "cs_core_eligible": 7110289,
63+ "cs_penumbra_eligible_otherwise": 3402475,
58 "eligible_by_period": { 64 "eligible_by_period": {
59− "pre": 1367, 65+ "pre": 1864,
60− "post": 3457315, 66+ "post": 3704028,
61− "baseline": 2944556 67+ "baseline": 3404397
62 }, 68 },
63 "eligible_by_year": { 69 "eligible_by_year": {
64− "2018": 561564, 70+ "2018": 642661,
65− "2019": 600167, 71+ "2019": 695209,
66− "2020": 612005, 72+ "2020": 708854,
67− "2021": 604393, 73+ "2021": 710953,
68− "2022": 610670, 74+ "2022": 699308,
69− "2023": 678150, 75+ "2023": 784990,
70− "2024": 785101, 76+ "2024": 901406,
71− "2025": 973274, 77+ "2025": 1065569,
72− "2026": 977914 78+ "2026": 901339
73 }, 79 },
74 "eligible_by_type": { 80 "eligible_by_type": {
75− "article": 3192312, 81+ "article": 3656399,
76− "conference-paper": 1840499, 82+ "conference-paper": 2138761,
77− "preprint": 624265, 83+ "preprint": 618946,
78− "book-chapter": 335735, 84+ "book-chapter": 343079,
79− "dissertation": 258440, 85+ "dissertation": 237332,
80− "report": 54120, 86+ "book": 47825,
81− "book": 52139, 87+ "report": 33064,
82− "review": 25251, 88+ "review": 28323,
83− "other": 17387, 89+ "software-paper": 3542,
84− "software-paper": 2350, 90+ "other": 1661,
85− "book-review": 615, 91+ "book-review": 1197,
86− "data-paper": 125 92+ "data-paper": 160
87 }, 93 },
88 "eligible_membership": { 94 "eligible_membership": {
89− "in_dblp": 2640780, 95+ "in_dblp": 3608016,
90− "cs_primary": 5619140, 96+ "cs_primary": 5322535,
91− "arxiv_linked": 709460, 97+ "arxiv_linked": 802308,
92− "in_acl": 70078, 98+ "in_acl": 70033,
93− "has_abstract": 4867840, 99+ "has_abstract": 5385235,
94− "has_references": 3715037, 100+ "has_references": 4614417,
95− "retracted_flagged": 14412 101+ "retracted_flagged": 17094
102+ },
103+ "eligible_date_precision": {
104+ "month": 5703718,
105+ "year": 1406571
96 }, 106 },
97− "strict_core_dblp_and_cs_primary": 1868188 107+ "strict_core_dblp_and_cs_primary": 1832623
98 } 108 }
\ No newline at end of file
modified experiments/emerging_detection.py +1 −1
@@ -78,7 +78,7 @@ def main():
78 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); ap.add_argument("--last-month", default="2026-06") 78 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); ap.add_argument("--last-month", default="2026-06")
79 ap.add_argument("--horizon", type=int, default=12); a = ap.parse_args(); R = a.root 79 ap.add_argument("--horizon", type=int, default=12); a = ap.parse_args(); R = a.root
80 out = os.path.join(R, "results/emerging"); os.makedirs(out, exist_ok=True) 80 out = os.path.join(R, "results/emerging"); os.makedirs(out, exist_ok=True)
81− con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb")); con.execute("SET threads TO 20; SET memory_limit='60GB';") 81+ con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb")); con.execute("SET threads TO 20; SET memory_limit='" + os.environ.get("WP44_DUCKDB_MEM", "60GB") + "';")
82 P, C = build_panel(R, con, a.last_month); P.to_parquet(f"{out}/panel_volume.parquet"); C.to_parquet(f"{out}/panel_citations.parquet") 82 P, C = build_panel(R, con, a.last_month); P.to_parquet(f"{out}/panel_volume.parquet"); C.to_parquet(f"{out}/panel_citations.parquet")
83 months = list(P.columns); h = a.horizon 83 months = list(P.columns); h = a.horizon
84 def idx(m): return months.index(m) 84 def idx(m): return months.index(m)
modified ingestion/compact_tables.py +1 −1
@@ -5,7 +5,7 @@ hit the macOS open-file limit. Also compacts the slim table of all OpenAlex work
5 import duckdb, os, sys, time 5 import duckdb, os, sys, time
6 R = sys.argv[1]; suffix = sys.argv[2] if len(sys.argv) > 2 else "" # e.g. "_p2" for the second snapshot pass 6 R = sys.argv[1]; suffix = sys.argv[2] if len(sys.argv) > 2 else "" # e.g. "_p2" for the second snapshot pass
7 tables_dir = os.path.join(R, "data/processed/tables" + suffix); out = os.path.join(R, "data/processed/compact"); os.makedirs(out, exist_ok=True) 7 tables_dir = os.path.join(R, "data/processed/tables" + suffix); out = os.path.join(R, "data/processed/compact"); os.makedirs(out, exist_ok=True)
8−con = duckdb.connect(); con.execute("SET threads TO 24; SET memory_limit='70GB'; SET preserve_insertion_order=false;") 8+con = duckdb.connect(); con.execute("SET threads TO 24; SET memory_limit='" + os.environ.get("WP44_DUCKDB_MEM", "60GB") + "'; SET preserve_insertion_order=false;")
9 jobs = [(t + suffix, f"{tables_dir}/{t}/*.parquet") for t in ["works", "authorships", "work_topics", "work_keywords", "references", "abstracts", "locations", "grants", "counts_by_year"]] 9 jobs = [(t + suffix, f"{tables_dir}/{t}/*.parquet") for t in ["works", "authorships", "work_topics", "work_keywords", "references", "abstracts", "locations", "grants", "counts_by_year"]]
10 if not suffix: 10 if not suffix:
11 jobs.append(("slim_all_works", f"{R}/data/raw/openalex/works/slim/*.parquet")) 11 jobs.append(("slim_all_works", f"{R}/data/raw/openalex/works/slim/*.parquet"))
modified ingestion/link_sources.py +1 −1
@@ -13,7 +13,7 @@ import duckdb
13 def main(): 13 def main():
14 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); ap.add_argument("--db", default=None) 14 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); ap.add_argument("--db", default=None)
15 a = ap.parse_args(); R = a.root; db = a.db or os.path.join(R, "data/processed/wp44.duckdb") 15 a = ap.parse_args(); R = a.root; db = a.db or os.path.join(R, "data/processed/wp44.duckdb")
16− con = duckdb.connect(db); con.execute("SET threads TO 24; SET memory_limit='60GB'; SET preserve_insertion_order=false;") 16+ con = duckdb.connect(db); con.execute("SET threads TO 24; SET memory_limit='" + os.environ.get("WP44_DUCKDB_MEM", "60GB") + "'; SET preserve_insertion_order=false;")
17 T = os.path.join(R, "data/processed/tables"); t0 = time.time(); rep = {"created": dt.datetime.now(dt.UTC).isoformat()} 17 T = os.path.join(R, "data/processed/tables"); t0 = time.time(); rep = {"created": dt.datetime.now(dt.UTC).isoformat()}
18 def log(m): print(f"[{time.time()-t0:7.0f}s] {m}", flush=True) 18 def log(m): print(f"[{time.time()-t0:7.0f}s] {m}", flush=True)
19 for t in ["works", "authorships", "work_topics", "work_keywords", "references", "abstracts", "locations", "grants", "counts_by_year"]: 19 for t in ["works", "authorships", "work_topics", "work_keywords", "references", "abstracts", "locations", "grants", "counts_by_year"]:
modified knowledge_graph/01_citation_graph.py +1 −1
@@ -15,7 +15,7 @@ import argparse, os, duckdb
15 def main(): 15 def main():
16 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); a = ap.parse_args(); R = a.root 16 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); a = ap.parse_args(); R = a.root
17 G = os.path.join(R, "data/processed/graph"); out = os.path.join(R, "results/graph"); os.makedirs(G, exist_ok=True); os.makedirs(out, exist_ok=True) 17 G = os.path.join(R, "data/processed/graph"); out = os.path.join(R, "results/graph"); os.makedirs(G, exist_ok=True); os.makedirs(out, exist_ok=True)
18− con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb")); con.execute("SET threads TO 24; SET memory_limit='70GB';") 18+ con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb")); con.execute("SET threads TO 24; SET memory_limit='" + os.environ.get("WP44_DUCKDB_MEM", "60GB") + "';")
19 con.execute("CREATE OR REPLACE TEMP TABLE E AS SELECT id, publication_year AS y, type, primary_field_id FROM corpus_works WHERE eligible") 19 con.execute("CREATE OR REPLACE TEMP TABLE E AS SELECT id, publication_year AS y, type, primary_field_id FROM corpus_works WHERE eligible")
20 con.execute("""CREATE OR REPLACE TABLE cit_edges_internal AS 20 con.execute("""CREATE OR REPLACE TABLE cit_edges_internal AS
21 SELECT r.work_id AS citing_id, r.ref_id AS cited_id, a.y AS citing_year, b.y AS cited_year 21 SELECT r.work_id AS citing_id, r.ref_id AS cited_id, a.y AS citing_year, b.y AS cited_year
modified knowledge_graph/02_category_flows.py +1 −1
@@ -21,7 +21,7 @@ def main():
21 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); ap.add_argument("--ontology", required=True); a = ap.parse_args(); R = a.root 21 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); ap.add_argument("--ontology", required=True); a = ap.parse_args(); R = a.root
22 out = os.path.join(R, "results/graph"); os.makedirs(out, exist_ok=True) 22 out = os.path.join(R, "results/graph"); os.makedirs(out, exist_ok=True)
23 ont = yaml.safe_load(open(a.ontology)); cat_area = {c["id"]: c["area"] for c in ont["categories"]}; cats = [c["id"] for c in ont["categories"]] 23 ont = yaml.safe_load(open(a.ontology)); cat_area = {c["id"]: c["area"] for c in ont["categories"]}; cats = [c["id"] for c in ont["categories"]]
24− con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb")); con.execute("SET threads TO 24; SET memory_limit='70GB';") 24+ con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb")); con.execute("SET threads TO 24; SET memory_limit='" + os.environ.get("WP44_DUCKDB_MEM", "60GB") + "';")
25 con.execute(f"CREATE OR REPLACE VIEW work_primary AS SELECT * FROM read_parquet('{R}/data/processed/classification/primary/*.parquet')") 25 con.execute(f"CREATE OR REPLACE VIEW work_primary AS SELECT * FROM read_parquet('{R}/data/processed/classification/primary/*.parquet')")
26 con.execute(f"CREATE OR REPLACE VIEW work_categories AS SELECT * FROM read_parquet('{R}/data/processed/classification/cats/*.parquet')") 26 con.execute(f"CREATE OR REPLACE VIEW work_categories AS SELECT * FROM read_parquet('{R}/data/processed/classification/cats/*.parquet')")
27 con.execute("CREATE OR REPLACE TEMP TABLE area_map AS SELECT * FROM (VALUES " + ",".join(f"('{c}','{cat_area[c]}')" for c in cats) + ") t(category_id, area)") 27 con.execute("CREATE OR REPLACE TEMP TABLE area_map AS SELECT * FROM (VALUES " + ",".join(f"('{c}','{cat_area[c]}')" for c in cats) + ") t(category_id, area)")
modified manuscript/tables/numbers.tex +98 −93
@@ -1,111 +1,116 @@
1 % machine-generated by tables/make_numbers.py — do not edit 1 % machine-generated by tables/make_numbers.py — do not edit
2−\newcommand{\PrismaGenerated}{2026-11-03T21:08:38.910490+00:00} 2+\newcommand{\PrismaGenerated}{2026-11-03T23:03:34.908225+00:00}
3 \newcommand{\PrismaBoundary}{2022-11-30} 3 \newcommand{\PrismaBoundary}{2022-11-30}
4−\newcommand{\PrismaIdentifiedOpenalexCsTopicTwoZeroOneEightTwoZeroTwoSixPassOne}{15{,}490{,}159} 4+\newcommand{\PrismaIdentifiedOpenalexCsTopicTwoZeroOneEightTwoZeroTwoSixPassOne}{16{,}592{,}966}
5−\newcommand{\PrismaIdentifiedOpenalexTotalRecordsLoaded}{15{,}490{,}159} 5+\newcommand{\PrismaIdentifiedOpenalexTotalRecordsLoaded}{16{,}592{,}966}
6 \newcommand{\PrismaIdentifiedDblpTwoZeroOneEightTwoZeroTwoSix}{4{,}347{,}365} 6 \newcommand{\PrismaIdentifiedDblpTwoZeroOneEightTwoZeroTwoSix}{4{,}347{,}365}
7 \newcommand{\PrismaIdentifiedArxivCsTwoZeroOneEightTwoZeroTwoSix}{890{,}852} 7 \newcommand{\PrismaIdentifiedArxivCsTwoZeroOneEightTwoZeroTwoSix}{890{,}852}
8 \newcommand{\PrismaIdentifiedAclTwoZeroOneEightTwoZeroTwoSix}{85{,}625} 8 \newcommand{\PrismaIdentifiedAclTwoZeroOneEightTwoZeroTwoSix}{85{,}625}
9−\newcommand{\PrismaLinkedDblpToOpenalex}{2{,}736{,}903} 9+\newcommand{\PrismaLinkedDblpToOpenalex}{3{,}774{,}184}
10−\newcommand{\PrismaLinkedArxivToOpenalex}{744{,}680} 10+\newcommand{\PrismaLinkedArxivToOpenalex}{864{,}677}
11−\newcommand{\PrismaLinkedAclToOpenalex}{72{,}256} 11+\newcommand{\PrismaLinkedAclToOpenalex}{73{,}054}
12−\newcommand{\PrismaLinkedDblpOnlyNoOpenalexRecord}{1{,}610{,}462} 12+\newcommand{\PrismaLinkedDblpOnlyNoOpenalexRecord}{573{,}181}
13−\newcommand{\PrismaExcludedEOneOutsideWindow}{160} 13+\newcommand{\PrismaExcludedEOneOutsideWindow}{59{,}453}
14−\newcommand{\PrismaExcludedETwoType}{3{,}961{,}633} 14+\newcommand{\PrismaExcludedETwoType}{3{,}995{,}235}
15−\newcommand{\PrismaExcludedEThreeXpac}{3{,}950{,}876} 15+\newcommand{\PrismaExcludedEThreeXpac}{3{,}974{,}822}
16−\newcommand{\PrismaExcludedEFourNoTitle}{16{,}426} 16+\newcommand{\PrismaExcludedEFourNoTitle}{16{,}442}
17−\newcommand{\PrismaExcludedEFiveDuplicate}{114{,}115} 17+\newcommand{\PrismaExcludedEFiveDuplicate}{120{,}147}
18−\newcommand{\PrismaExcludedESixRetracted}{25{,}165} 18+\newcommand{\PrismaExcludedESevenGenericDeposit}{1{,}277{,}584}
19−\newcommand{\PrismaExcludedXpacKeptDblp}{87{,}288} 19+\newcommand{\PrismaExcludedESixRetracted}{28{,}587}
20−\newcommand{\PrismaExcludedETwoByTypeDataset}{2{,}775{,}759} 20+\newcommand{\PrismaExcludedXpacKeptDblp}{74{,}771}
21−\newcommand{\PrismaExcludedETwoByTypeOther}{522{,}654} 21+\newcommand{\PrismaExcludedETwoByTypeDataset}{2{,}779{,}058}
22+\newcommand{\PrismaExcludedETwoByTypeOther}{538{,}613}
22 \newcommand{\PrismaExcludedETwoByTypeLibguides}{191{,}658} 23 \newcommand{\PrismaExcludedETwoByTypeLibguides}{191{,}658}
23−\newcommand{\PrismaExcludedETwoByTypeSoftware}{177{,}500} 24+\newcommand{\PrismaExcludedETwoByTypeSoftware}{177{,}515}
24−\newcommand{\PrismaExcludedETwoByTypeConferenceAbstract}{66{,}267} 25+\newcommand{\PrismaExcludedETwoByTypeConferenceAbstract}{68{,}101}
25−\newcommand{\PrismaExcludedETwoByTypePeerReview}{60{,}176} 26+\newcommand{\PrismaExcludedETwoByTypePeerReview}{60{,}180}
26−\newcommand{\PrismaExcludedETwoByTypeParatext}{39{,}162} 27+\newcommand{\PrismaExcludedETwoByTypeParatext}{41{,}468}
27−\newcommand{\PrismaExcludedETwoByTypeReferenceEntry}{32{,}247} 28+\newcommand{\PrismaExcludedETwoByTypeReferenceEntry}{33{,}290}
28−\newcommand{\PrismaExcludedETwoByTypeEditorial}{24{,}907} 29+\newcommand{\PrismaExcludedETwoByTypeEditorial}{29{,}543}
29−\newcommand{\PrismaExcludedETwoByTypeSupplementaryMaterials}{18{,}882} 30+\newcommand{\PrismaExcludedETwoByTypeErratum}{20{,}850}
30−\newcommand{\PrismaExcludedETwoByTypeErratum}{17{,}340} 31+\newcommand{\PrismaExcludedETwoByTypeSupplementaryMaterials}{18{,}946}
31−\newcommand{\PrismaExcludedETwoByTypeStandard}{10{,}113} 32+\newcommand{\PrismaExcludedETwoByTypeStandard}{10{,}125}
32−\newcommand{\PrismaExcludedETwoByTypeBookReview}{9{,}517} 33+\newcommand{\PrismaExcludedETwoByTypeBookReview}{9{,}524}
33−\newcommand{\PrismaExcludedETwoByTypeRetraction}{8{,}217} 34+\newcommand{\PrismaExcludedETwoByTypeRetraction}{8{,}924}
34−\newcommand{\PrismaExcludedETwoByTypeDataPaper}{5{,}255} 35+\newcommand{\PrismaExcludedETwoByTypeDataPaper}{5{,}308}
35−\newcommand{\PrismaExcludedETwoByTypeLetter}{1{,}979} 36+\newcommand{\PrismaExcludedETwoByTypeLetter}{2{,}132}
36−\newcommand{\PrismaExcludedSequentialEOneOutsideWindow}{160} 37+\newcommand{\PrismaExcludedSequentialEOneOutsideWindow}{59{,}453}
37−\newcommand{\PrismaExcludedSequentialETwoType}{3{,}961{,}589} 38+\newcommand{\PrismaExcludedSequentialETwoType}{3{,}994{,}472}
38−\newcommand{\PrismaExcludedSequentialEThreeXpac}{1{,}421{,}692} 39+\newcommand{\PrismaExcludedSequentialEThreeXpac}{1{,}430{,}851}
39−\newcommand{\PrismaExcludedSequentialEFourNoTitle}{1{,}188} 40+\newcommand{\PrismaExcludedSequentialEFourNoTitle}{1{,}193}
40−\newcommand{\PrismaExcludedSequentialEFiveDuplicate}{106{,}928} 41+\newcommand{\PrismaExcludedSequentialEFiveDuplicate}{114{,}421}
41−\newcommand{\PrismaAfterExclusions}{9{,}998{,}602} 42+\newcommand{\PrismaExcludedSequentialESevenGenericDeposit}{473{,}549}
42−\newcommand{\PrismaCsCoreEligible}{6{,}403{,}238} 43+\newcommand{\PrismaAfterExclusions}{10{,}519{,}027}
43−\newcommand{\PrismaCsPenumbraEligibleOtherwise}{3{,}595{,}364} 44+\newcommand{\PrismaVersionRelationshipArxivId}{13{,}175}
44−\newcommand{\PrismaEligibleByPeriodPre}{1{,}367} 45+\newcommand{\PrismaVersionRelationshipTitleYear}{84{,}776}
45−\newcommand{\PrismaEligibleByPeriodPost}{3{,}457{,}315} 46+\newcommand{\PrismaCsCoreEligible}{7{,}110{,}289}
46−\newcommand{\PrismaEligibleByPeriodBaseline}{2{,}944{,}556} 47+\newcommand{\PrismaCsPenumbraEligibleOtherwise}{3{,}402{,}475}
47−\newcommand{\PrismaEligibleByYearTwoZeroOneEight}{561{,}564} 48+\newcommand{\PrismaEligibleByPeriodPre}{1{,}864}
48−\newcommand{\PrismaEligibleByYearTwoZeroOneNine}{600{,}167} 49+\newcommand{\PrismaEligibleByPeriodPost}{3{,}704{,}028}
49−\newcommand{\PrismaEligibleByYearTwoZeroTwoZero}{612{,}005} 50+\newcommand{\PrismaEligibleByPeriodBaseline}{3{,}404{,}397}
50−\newcommand{\PrismaEligibleByYearTwoZeroTwoOne}{604{,}393} 51+\newcommand{\PrismaEligibleByYearTwoZeroOneEight}{642{,}661}
51−\newcommand{\PrismaEligibleByYearTwoZeroTwoTwo}{610{,}670} 52+\newcommand{\PrismaEligibleByYearTwoZeroOneNine}{695{,}209}
52−\newcommand{\PrismaEligibleByYearTwoZeroTwoThree}{678{,}150} 53+\newcommand{\PrismaEligibleByYearTwoZeroTwoZero}{708{,}854}
53−\newcommand{\PrismaEligibleByYearTwoZeroTwoFour}{785{,}101} 54+\newcommand{\PrismaEligibleByYearTwoZeroTwoOne}{710{,}953}
54−\newcommand{\PrismaEligibleByYearTwoZeroTwoFive}{973{,}274} 55+\newcommand{\PrismaEligibleByYearTwoZeroTwoTwo}{699{,}308}
55−\newcommand{\PrismaEligibleByYearTwoZeroTwoSix}{977{,}914} 56+\newcommand{\PrismaEligibleByYearTwoZeroTwoThree}{784{,}990}
56−\newcommand{\PrismaEligibleByTypeArticle}{3{,}192{,}312} 57+\newcommand{\PrismaEligibleByYearTwoZeroTwoFour}{901{,}406}
57−\newcommand{\PrismaEligibleByTypeConferencePaper}{1{,}840{,}499} 58+\newcommand{\PrismaEligibleByYearTwoZeroTwoFive}{1{,}065{,}569}
58−\newcommand{\PrismaEligibleByTypePreprint}{624{,}265} 59+\newcommand{\PrismaEligibleByYearTwoZeroTwoSix}{901{,}339}
59−\newcommand{\PrismaEligibleByTypeBookChapter}{335{,}735} 60+\newcommand{\PrismaEligibleByTypeArticle}{3{,}656{,}399}
60−\newcommand{\PrismaEligibleByTypeDissertation}{258{,}440} 61+\newcommand{\PrismaEligibleByTypeConferencePaper}{2{,}138{,}761}
61−\newcommand{\PrismaEligibleByTypeReport}{54{,}120} 62+\newcommand{\PrismaEligibleByTypePreprint}{618{,}946}
62−\newcommand{\PrismaEligibleByTypeBook}{52{,}139} 63+\newcommand{\PrismaEligibleByTypeBookChapter}{343{,}079}
63−\newcommand{\PrismaEligibleByTypeReview}{25{,}251} 64+\newcommand{\PrismaEligibleByTypeDissertation}{237{,}332}
64−\newcommand{\PrismaEligibleByTypeOther}{17{,}387} 65+\newcommand{\PrismaEligibleByTypeBook}{47{,}825}
65−\newcommand{\PrismaEligibleByTypeSoftwarePaper}{2{,}350} 66+\newcommand{\PrismaEligibleByTypeReport}{33{,}064}
66−\newcommand{\PrismaEligibleByTypeBookReview}{615} 67+\newcommand{\PrismaEligibleByTypeReview}{28{,}323}
67−\newcommand{\PrismaEligibleByTypeDataPaper}{125} 68+\newcommand{\PrismaEligibleByTypeSoftwarePaper}{3{,}542}
68−\newcommand{\PrismaEligibleMembershipInDblp}{2{,}640{,}780} 69+\newcommand{\PrismaEligibleByTypeOther}{1{,}661}
69−\newcommand{\PrismaEligibleMembershipCsPrimary}{5{,}619{,}140} 70+\newcommand{\PrismaEligibleByTypeBookReview}{1{,}197}
70−\newcommand{\PrismaEligibleMembershipArxivLinked}{709{,}460} 71+\newcommand{\PrismaEligibleByTypeDataPaper}{160}
71−\newcommand{\PrismaEligibleMembershipInAcl}{70{,}078} 72+\newcommand{\PrismaEligibleMembershipInDblp}{3{,}608{,}016}
72−\newcommand{\PrismaEligibleMembershipHasAbstract}{4{,}867{,}840} 73+\newcommand{\PrismaEligibleMembershipCsPrimary}{5{,}322{,}535}
73−\newcommand{\PrismaEligibleMembershipHasReferences}{3{,}715{,}037} 74+\newcommand{\PrismaEligibleMembershipArxivLinked}{802{,}308}
74−\newcommand{\PrismaEligibleMembershipRetractedFlagged}{14{,}412} 75+\newcommand{\PrismaEligibleMembershipInAcl}{70{,}033}
75−\newcommand{\PrismaStrictCoreDblpAndCsPrimary}{1{,}868{,}188} 76+\newcommand{\PrismaEligibleMembershipHasAbstract}{5{,}385{,}235}
76−\newcommand{\LinkCreated}{2026-11-03T20:50:56.182276+00:00} 77+\newcommand{\PrismaEligibleMembershipHasReferences}{4{,}614{,}417}
77−\newcommand{\LinkOaCsRecords}{15{,}490{,}159} 78+\newcommand{\PrismaEligibleMembershipRetractedFlagged}{17{,}094}
78−\newcommand{\LinkOaCsDistinctIds}{15{,}490{,}159} 79+\newcommand{\PrismaEligibleDatePrecisionMonth}{5{,}703{,}718}
80+\newcommand{\PrismaEligibleDatePrecisionYear}{1{,}406{,}571}
81+\newcommand{\PrismaStrictCoreDblpAndCsPrimary}{1{,}832{,}623}
82+\newcommand{\LinkCreated}{2026-11-03T23:03:10.160666+00:00}
83+\newcommand{\LinkOaCsRecords}{16{,}592{,}966}
84+\newcommand{\LinkOaCsDistinctIds}{16{,}592{,}966}
79 \newcommand{\LinkDblpTwoZeroOneEightTwoZeroTwoSix}{4{,}347{,}365} 85 \newcommand{\LinkDblpTwoZeroOneEightTwoZeroTwoSix}{4{,}347{,}365}
80 \newcommand{\LinkDblpWithDoi}{4{,}052{,}326} 86 \newcommand{\LinkDblpWithDoi}{4{,}052{,}326}
81 \newcommand{\LinkDblpWithArxiv}{0} 87 \newcommand{\LinkDblpWithArxiv}{0}
82−\newcommand{\LinkLinkDblpOaTitleYear}{179{,}990} 88+\newcommand{\LinkLinkDblpOaTitleYear}{141{,}062}
83−\newcommand{\LinkLinkDblpOaDoi}{2{,}562{,}431} 89+\newcommand{\LinkLinkDblpOaDoi}{3{,}639{,}848}
84−\newcommand{\LinkDblpLinkedDistinct}{2{,}736{,}903} 90+\newcommand{\LinkDblpLinkedDistinct}{3{,}774{,}184}
85−\newcommand{\LinkDblpUnlinked}{1{,}610{,}462} 91+\newcommand{\LinkDblpUnlinked}{573{,}181}
86−\newcommand{\LinkDblpUnlinkedByTypePhdthesis}{26{,}841} 92+\newcommand{\LinkDblpUnlinkedByTypeIncollection}{418}
87−\newcommand{\LinkDblpUnlinkedByTypeIncollection}{3{,}362} 93+\newcommand{\LinkDblpUnlinkedByTypeBook}{629}
88−\newcommand{\LinkDblpUnlinkedByTypeInproceedings}{477{,}235} 94+\newcommand{\LinkDblpUnlinkedByTypePhdthesis}{30{,}292}
89−\newcommand{\LinkDblpUnlinkedByTypeBook}{1{,}398} 95+\newcommand{\LinkDblpUnlinkedByTypeInproceedings}{159{,}308}
90−\newcommand{\LinkDblpUnlinkedByTypeArticle}{1{,}084{,}640} 96+\newcommand{\LinkDblpUnlinkedByTypeArticle}{367{,}850}
91−\newcommand{\LinkDblpUnlinkedByTypeData}{16{,}986} 97+\newcommand{\LinkDblpUnlinkedByTypeData}{14{,}684}
92−\newcommand{\LinkDblpUnlinkedWithDoiInSlim}{1{,}077{,}407} 98+\newcommand{\LinkDblpUnlinkedWithDoiInSlim}{1{,}605}
93 \newcommand{\LinkArxivCsRecords}{915{,}115} 99 \newcommand{\LinkArxivCsRecords}{915{,}115}
94 \newcommand{\LinkArxivCreatedTwoZeroOneEightTwoZeroTwoSix}{890{,}852} 100 \newcommand{\LinkArxivCreatedTwoZeroOneEightTwoZeroTwoSix}{890{,}852}
95−\newcommand{\LinkLinkArxivOaArxivId}{792{,}672} 101+\newcommand{\LinkLinkArxivOaTitleYear}{1{,}768}
96−\newcommand{\LinkLinkArxivOaDoi}{520} 102+\newcommand{\LinkLinkArxivOaDoi}{783}
97−\newcommand{\LinkLinkArxivOaTitleYear}{2{,}143} 103+\newcommand{\LinkLinkArxivOaArxivId}{914{,}406}
98−\newcommand{\LinkArxivLinkedDistinct}{744{,}680} 104+\newcommand{\LinkArxivLinkedDistinct}{864{,}677}
99 \newcommand{\LinkAclRecords}{135{,}558} 105 \newcommand{\LinkAclRecords}{135{,}558}
100 \newcommand{\LinkAclTwoZeroOneEightTwoZeroTwoSix}{85{,}625} 106 \newcommand{\LinkAclTwoZeroOneEightTwoZeroTwoSix}{85{,}625}
101−\newcommand{\LinkLinkAclOaDoi}{57{,}474} 107+\newcommand{\LinkLinkAclOaDoi}{58{,}174}
102−\newcommand{\LinkLinkAclOaTitleYear}{14{,}904} 108+\newcommand{\LinkLinkAclOaTitleYear}{15{,}004}
103−\newcommand{\LinkAclLinkedDistinctTwoZeroOneEightTwoZeroTwoSix}{72{,}223} 109+\newcommand{\LinkAclLinkedDistinctTwoZeroOneEightTwoZeroTwoSix}{72{,}949}
104−\newcommand{\LinkNeededOaIdsSecondPassDblpDoi}{1{,}077{,}406} 110+\newcommand{\LinkNeededOaIdsSecondPassDblpDoi}{1{,}605}
105−\newcommand{\LinkNeededOaIdsSecondPassArxivDoi}{39{,}200}
106 \newcommand{\LinkSlimTotal}{464{,}765{,}485} 111 \newcommand{\LinkSlimTotal}{464{,}765{,}485}
107 \newcommand{\LinkSlimCsAnyTwoZeroOneEightTwoZeroTwoSix}{15{,}490{,}159} 112 \newcommand{\LinkSlimCsAnyTwoZeroOneEightTwoZeroTwoSix}{15{,}490{,}159}
108−\newcommand{\LinkSeconds}{10} 113+\newcommand{\LinkSeconds}{12}
109 \newcommand{\DblpParsedRecords}{8{,}840{,}866} 114 \newcommand{\DblpParsedRecords}{8{,}840{,}866}
110 \newcommand{\DblpTypeIncollection}{71{,}247} 115 \newcommand{\DblpTypeIncollection}{71{,}247}
111 \newcommand{\DblpTypeBook}{21{,}648} 116 \newcommand{\DblpTypeBook}{21{,}648}
modified systematic_review/define_corpus.py +1 −1
@@ -18,7 +18,7 @@ WINDOW_START, BOUNDARY, WINDOW_END = "2018-01-01", "2022-11-30", "2026-11-02"
18 18
19 def main(): 19 def main():
20 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); a = ap.parse_args(); R = a.root 20 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); a = ap.parse_args(); R = a.root
21− con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb")); con.execute("SET threads TO 24; SET memory_limit='60GB';") 21+ con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb")); con.execute("SET threads TO 24; SET memory_limit='" + os.environ.get("WP44_DUCKDB_MEM", "60GB") + "';")
22 t0 = time.time(); q = lambda s: con.execute(s).fetchall(); n = lambda s: q(s)[0][0] 22 t0 = time.time(); q = lambda s: con.execute(s).fetchall(); n = lambda s: q(s)[0][0]
23 log = lambda m: print(f"[{time.time()-t0:6.0f}s] {m}", flush=True) 23 log = lambda m: print(f"[{time.time()-t0:6.0f}s] {m}", flush=True)
24 # ---- S0/S1: base table with membership flags ---- 24 # ---- S0/S1: base table with membership flags ----
modified taxonomy/classify.py +1 −1
@@ -95,7 +95,7 @@ def process_chunk(args):
95 95
96 96
97 def prepare(root, db, out_dir, chunk_rows): 97 def prepare(root, db, out_dir, chunk_rows):
98− con = duckdb.connect(db); con.execute("SET threads TO 24; SET memory_limit='60GB';") 98+ con = duckdb.connect(db); con.execute("SET threads TO 24; SET memory_limit='" + os.environ.get("WP44_DUCKDB_MEM", "60GB") + "';")
99 os.makedirs(os.path.join(out_dir, "input"), exist_ok=True) 99 os.makedirs(os.path.join(out_dir, "input"), exist_ok=True)
100 con.execute("""CREATE OR REPLACE TABLE cls_input AS 100 con.execute("""CREATE OR REPLACE TABLE cls_input AS
101 SELECT w.id, w.publication_year AS year, w.title, a.abstract, 101 SELECT w.id, w.publication_year AS year, w.title, a.abstract,
102 102
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere