SPB Git
spboucher.ai

spb/wp44_uqo

Public

WP44: corpus v2 metadata (7.11 M core works), configurable DuckDB memory

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Nov 3, 2026) parent f781212 Browse files

13 changed files +263 −195

13 changed files
Unified Split
modified bibliometrics/02_collaboration.py +1 −1
@@ -10,7 +10,7 @@ import argparse, os, duckdb
10 10 def main():
11 11 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); a = ap.parse_args(); R = a.root
12 12 out = os.path.join(R, "results/bibliometrics"); os.makedirs(out, exist_ok=True)
13 − con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb"), read_only=True); con.execute("SET threads TO 20; SET memory_limit='60GB';")
13 + con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb"), read_only=True); con.execute("SET threads TO 20; SET memory_limit='" + os.environ.get("WP44_DUCKDB_MEM", "60GB") + "';")
14 14 con.execute("CREATE OR REPLACE TEMP VIEW E AS SELECT * FROM corpus_works WHERE eligible")
15 15 con.execute(f"""COPY (SELECT publication_year AS year, period, count(*) n, round(avg(n_authors),3) mean_authors, quantile_cont(n_authors,0.5) med_authors,
16 16 sum((n_authors=1)::int) single, sum((n_authors>=5)::int) five_plus, sum((n_authors>=10)::int) ten_plus, sum((n_countries>=2)::int) intl,
modified bibliometrics/03_citation_impact.py +1 −1
@@ -14,7 +14,7 @@ import argparse, os, duckdb
14 14 def main():
15 15 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); a = ap.parse_args(); R = a.root
16 16 out = os.path.join(R, "results/bibliometrics"); os.makedirs(out, exist_ok=True)
17 − con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb")); con.execute("SET threads TO 20; SET memory_limit='60GB';")
17 + con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb")); con.execute("SET threads TO 20; SET memory_limit='" + os.environ.get("WP44_DUCKDB_MEM", "60GB") + "';")
18 18 con.execute("""CREATE OR REPLACE TABLE work_citation_windows AS
19 19 WITH E AS (SELECT id, preprint_work_id, publication_year AS y, coalesce(type, 'unknown') AS type, coalesce(primary_subfield_id, -1) AS sf, cited_by_count, fwci
20 20 FROM corpus_works WHERE eligible AND NOT e6_retracted),
added data/metadata/dblp_only_counts.csv +54 −0
@@ -0,0 +1,54 @@
1 +year,rec_type,n,with_doi
2 +2018,article,22769,19614
3 +2018,book,89,4
4 +2018,data,246,245
5 +2018,incollection,171,115
6 +2018,inproceedings,11322,508
7 +2018,phdthesis,4430,43
8 +2019,article,29350,26379
9 +2019,book,115,4
10 +2019,data,471,444
11 +2019,incollection,30,5
12 +2019,inproceedings,11656,546
13 +2019,phdthesis,4162,35
14 +2020,article,37639,34805
15 +2020,book,62,5
16 +2020,data,816,807
17 +2020,incollection,52,5
18 +2020,inproceedings,13639,610
19 +2020,phdthesis,3857,79
20 +2021,article,39300,36294
21 +2021,book,80,5
22 +2021,data,972,927
23 +2021,incollection,47,0
24 +2021,inproceedings,15423,588
25 +2021,phdthesis,4222,86
26 +2022,article,46967,43149
27 +2022,book,70,1
28 +2022,data,1335,1265
29 +2022,incollection,33,2
30 +2022,inproceedings,19180,574
31 +2022,phdthesis,4133,138
32 +2023,article,56159,51893
33 +2023,book,88,1
34 +2023,data,1773,1691
35 +2023,incollection,21,21
36 +2023,inproceedings,18807,827
37 +2023,phdthesis,3648,97
38 +2024,article,66262,61366
39 +2024,book,60,2
40 +2024,data,2471,2372
41 +2024,incollection,23,0
42 +2024,inproceedings,24391,646
43 +2024,phdthesis,3343,69
44 +2025,article,53124,47735
45 +2025,book,44,0
46 +2025,data,3835,3782
47 +2025,incollection,41,0
48 +2025,inproceedings,26050,692
49 +2025,phdthesis,2099,1
50 +2026,article,16280,11901
51 +2026,book,21,2
52 +2026,data,2765,2741
53 +2026,inproceedings,18840,3473
54 +2026,phdthesis,398,0
modified data/metadata/linkage_report.json +23 −24
@@ -1,45 +1,44 @@
1 1 {
2 − "created": "2026-11-03T20:50:56.182276+00:00",
3 − "oa_cs_records": 15490159,
4 − "oa_cs_distinct_ids": 15490159,
2 + "created": "2026-11-03T23:03:10.160666+00:00",
3 + "oa_cs_records": 16592966,
4 + "oa_cs_distinct_ids": 16592966,
5 5 "dblp_2018_2026": 4347365,
6 6 "dblp_with_doi": 4052326,
7 7 "dblp_with_arxiv": 0,
8 8 "link_dblp_oa": {
9 − "title_year": 179990,
10 − "doi": 2562431
9 + "title_year": 141062,
10 + "doi": 3639848
11 11 },
12 − "dblp_linked_distinct": 2736903,
13 − "dblp_unlinked": 1610462,
12 + "dblp_linked_distinct": 3774184,
13 + "dblp_unlinked": 573181,
14 14 "dblp_unlinked_by_type": {
15 − "phdthesis": 26841,
16 − "incollection": 3362,
17 − "inproceedings": 477235,
18 − "book": 1398,
19 − "article": 1084640,
20 − "data": 16986
15 + "incollection": 418,
16 + "book": 629,
17 + "phdthesis": 30292,
18 + "inproceedings": 159308,
19 + "article": 367850,
20 + "data": 14684
21 21 },
22 − "dblp_unlinked_with_doi_in_slim": 1077407,
22 + "dblp_unlinked_with_doi_in_slim": 1605,
23 23 "arxiv_cs_records": 915115,
24 24 "arxiv_created_2018_2026": 890852,
25 25 "link_arxiv_oa": {
26 − "arxiv_id": 792672,
27 − "doi": 520,
28 − "title_year": 2143
26 + "title_year": 1768,
27 + "doi": 783,
28 + "arxiv_id": 914406
29 29 },
30 − "arxiv_linked_distinct": 744680,
30 + "arxiv_linked_distinct": 864677,
31 31 "acl_records": 135558,
32 32 "acl_2018_2026": 85625,
33 33 "link_acl_oa": {
34 − "doi": 57474,
35 − "title_year": 14904
34 + "doi": 58174,
35 + "title_year": 15004
36 36 },
37 − "acl_linked_distinct_2018_2026": 72223,
37 + "acl_linked_distinct_2018_2026": 72949,
38 38 "needed_oa_ids_second_pass": {
39 − "dblp_doi": 1077406,
40 − "arxiv_doi": 39200
39 + "dblp_doi": 1605
41 40 },
42 41 "slim_total": 464765485,
43 42 "slim_cs_any_2018_2026": 15490159,
44 − "seconds": 10
43 + "seconds": 12
45 44 }
\ No newline at end of file
modified data/metadata/prisma_counts.json +79 −69
@@ -1,98 +1,108 @@
1 1 {
2 − "generated": "2026-11-03T21:08:38.910490+00:00",
2 + "generated": "2026-11-03T23:03:34.908225+00:00",
3 3 "window": [
4 4 "2018-01-01",
5 5 "2026-11-02"
6 6 ],
7 7 "boundary": "2022-11-30",
8 8 "identified": {
9 − "openalex_cs_topic_2018_2026_pass1": 15490159,
10 − "openalex_total_records_loaded": 15490159,
9 + "openalex_cs_topic_2018_2026_pass1": 16592966,
10 + "openalex_total_records_loaded": 16592966,
11 11 "dblp_2018_2026": 4347365,
12 12 "arxiv_cs_2018_2026": 890852,
13 13 "acl_2018_2026": 85625
14 14 },
15 15 "linked": {
16 − "dblp_to_openalex": 2736903,
17 − "arxiv_to_openalex": 744680,
18 − "acl_to_openalex": 72256,
19 − "dblp_only_no_openalex_record": 1610462
16 + "dblp_to_openalex": 3774184,
17 + "arxiv_to_openalex": 864677,
18 + "acl_to_openalex": 73054,
19 + "dblp_only_no_openalex_record": 573181
20 20 },
21 21 "excluded": {
22 − "e1_outside_window": 160,
23 − "e2_type": 3961633,
24 − "e3_xpac": 3950876,
25 − "e4_no_title": 16426,
26 − "e5_duplicate": 114115,
27 − "e6_retracted": 25165,
28 − "xpac_kept_dblp": 87288
22 + "e1_outside_window": 59453,
23 + "e2_type": 3995235,
24 + "e3_xpac": 3974822,
25 + "e4_no_title": 16442,
26 + "e5_duplicate": 120147,
27 + "e7_generic_deposit": 1277584,
28 + "e6_retracted": 28587,
29 + "xpac_kept_dblp": 74771
29 30 },
30 31 "excluded_e2_by_type": {
31 − "dataset": 2775759,
32 − "other": 522654,
32 + "dataset": 2779058,
33 + "other": 538613,
33 34 "libguides": 191658,
34 − "software": 177500,
35 − "conference-abstract": 66267,
36 − "peer-review": 60176,
37 − "paratext": 39162,
38 − "reference-entry": 32247,
39 − "editorial": 24907,
40 − "supplementary-materials": 18882,
41 − "erratum": 17340,
42 − "standard": 10113,
43 − "book-review": 9517,
44 − "retraction": 8217,
45 − "data-paper": 5255,
46 − "letter": 1979
35 + "software": 177515,
36 + "conference-abstract": 68101,
37 + "peer-review": 60180,
38 + "paratext": 41468,
39 + "reference-entry": 33290,
40 + "editorial": 29543,
41 + "erratum": 20850,
42 + "supplementary-materials": 18946,
43 + "standard": 10125,
44 + "book-review": 9524,
45 + "retraction": 8924,
46 + "data-paper": 5308,
47 + "letter": 2132
47 48 },
48 49 "excluded_sequential": {
49 − "e1_outside_window": 160,
50 − "e2_type": 3961589,
51 − "e3_xpac": 1421692,
52 − "e4_no_title": 1188,
53 − "e5_duplicate": 106928
50 + "e1_outside_window": 59453,
51 + "e2_type": 3994472,
52 + "e3_xpac": 1430851,
53 + "e4_no_title": 1193,
54 + "e5_duplicate": 114421,
55 + "e7_generic_deposit": 473549
54 56 },
55 − "after_exclusions": 9998602,
56 − "cs_core_eligible": 6403238,
57 − "cs_penumbra_eligible_otherwise": 3595364,
57 + "after_exclusions": 10519027,
58 + "version_relationship": {
59 + "arxiv_id": 13175,
60 + "title_year": 84776
61 + },
62 + "cs_core_eligible": 7110289,
63 + "cs_penumbra_eligible_otherwise": 3402475,
58 64 "eligible_by_period": {
59 − "pre": 1367,
60 − "post": 3457315,
61 − "baseline": 2944556
65 + "pre": 1864,
66 + "post": 3704028,
67 + "baseline": 3404397
62 68 },
63 69 "eligible_by_year": {
64 − "2018": 561564,
65 − "2019": 600167,
66 − "2020": 612005,
67 − "2021": 604393,
68 − "2022": 610670,
69 − "2023": 678150,
70 − "2024": 785101,
71 − "2025": 973274,
72 − "2026": 977914
70 + "2018": 642661,
71 + "2019": 695209,
72 + "2020": 708854,
73 + "2021": 710953,
74 + "2022": 699308,
75 + "2023": 784990,
76 + "2024": 901406,
77 + "2025": 1065569,
78 + "2026": 901339
73 79 },
74 80 "eligible_by_type": {
75 − "article": 3192312,
76 − "conference-paper": 1840499,
77 − "preprint": 624265,
78 − "book-chapter": 335735,
79 − "dissertation": 258440,
80 − "report": 54120,
81 − "book": 52139,
82 − "review": 25251,
83 − "other": 17387,
84 − "software-paper": 2350,
85 − "book-review": 615,
86 − "data-paper": 125
81 + "article": 3656399,
82 + "conference-paper": 2138761,
83 + "preprint": 618946,
84 + "book-chapter": 343079,
85 + "dissertation": 237332,
86 + "book": 47825,
87 + "report": 33064,
88 + "review": 28323,
89 + "software-paper": 3542,
90 + "other": 1661,
91 + "book-review": 1197,
92 + "data-paper": 160
87 93 },
88 94 "eligible_membership": {
89 − "in_dblp": 2640780,
90 − "cs_primary": 5619140,
91 − "arxiv_linked": 709460,
92 − "in_acl": 70078,
93 − "has_abstract": 4867840,
94 − "has_references": 3715037,
95 − "retracted_flagged": 14412
95 + "in_dblp": 3608016,
96 + "cs_primary": 5322535,
97 + "arxiv_linked": 802308,
98 + "in_acl": 70033,
99 + "has_abstract": 5385235,
100 + "has_references": 4614417,
101 + "retracted_flagged": 17094
102 + },
103 + "eligible_date_precision": {
104 + "month": 5703718,
105 + "year": 1406571
96 106 },
97 − "strict_core_dblp_and_cs_primary": 1868188
107 + "strict_core_dblp_and_cs_primary": 1832623
98 108 }
\ No newline at end of file
modified experiments/emerging_detection.py +1 −1
@@ -78,7 +78,7 @@ def main():
78 78 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); ap.add_argument("--last-month", default="2026-06")
79 79 ap.add_argument("--horizon", type=int, default=12); a = ap.parse_args(); R = a.root
80 80 out = os.path.join(R, "results/emerging"); os.makedirs(out, exist_ok=True)
81 − con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb")); con.execute("SET threads TO 20; SET memory_limit='60GB';")
81 + con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb")); con.execute("SET threads TO 20; SET memory_limit='" + os.environ.get("WP44_DUCKDB_MEM", "60GB") + "';")
82 82 P, C = build_panel(R, con, a.last_month); P.to_parquet(f"{out}/panel_volume.parquet"); C.to_parquet(f"{out}/panel_citations.parquet")
83 83 months = list(P.columns); h = a.horizon
84 84 def idx(m): return months.index(m)
modified ingestion/compact_tables.py +1 −1
@@ -5,7 +5,7 @@ hit the macOS open-file limit. Also compacts the slim table of all OpenAlex work
5 5 import duckdb, os, sys, time
6 6 R = sys.argv[1]; suffix = sys.argv[2] if len(sys.argv) > 2 else "" # e.g. "_p2" for the second snapshot pass
7 7 tables_dir = os.path.join(R, "data/processed/tables" + suffix); out = os.path.join(R, "data/processed/compact"); os.makedirs(out, exist_ok=True)
8 −con = duckdb.connect(); con.execute("SET threads TO 24; SET memory_limit='70GB'; SET preserve_insertion_order=false;")
8 +con = duckdb.connect(); con.execute("SET threads TO 24; SET memory_limit='" + os.environ.get("WP44_DUCKDB_MEM", "60GB") + "'; SET preserve_insertion_order=false;")
9 9 jobs = [(t + suffix, f"{tables_dir}/{t}/*.parquet") for t in ["works", "authorships", "work_topics", "work_keywords", "references", "abstracts", "locations", "grants", "counts_by_year"]]
10 10 if not suffix:
11 11 jobs.append(("slim_all_works", f"{R}/data/raw/openalex/works/slim/*.parquet"))
modified ingestion/link_sources.py +1 −1
@@ -13,7 +13,7 @@ import duckdb
13 13 def main():
14 14 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); ap.add_argument("--db", default=None)
15 15 a = ap.parse_args(); R = a.root; db = a.db or os.path.join(R, "data/processed/wp44.duckdb")
16 − con = duckdb.connect(db); con.execute("SET threads TO 24; SET memory_limit='60GB'; SET preserve_insertion_order=false;")
16 + con = duckdb.connect(db); con.execute("SET threads TO 24; SET memory_limit='" + os.environ.get("WP44_DUCKDB_MEM", "60GB") + "'; SET preserve_insertion_order=false;")
17 17 T = os.path.join(R, "data/processed/tables"); t0 = time.time(); rep = {"created": dt.datetime.now(dt.UTC).isoformat()}
18 18 def log(m): print(f"[{time.time()-t0:7.0f}s] {m}", flush=True)
19 19 for t in ["works", "authorships", "work_topics", "work_keywords", "references", "abstracts", "locations", "grants", "counts_by_year"]:
modified knowledge_graph/01_citation_graph.py +1 −1
@@ -15,7 +15,7 @@ import argparse, os, duckdb
15 15 def main():
16 16 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); a = ap.parse_args(); R = a.root
17 17 G = os.path.join(R, "data/processed/graph"); out = os.path.join(R, "results/graph"); os.makedirs(G, exist_ok=True); os.makedirs(out, exist_ok=True)
18 − con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb")); con.execute("SET threads TO 24; SET memory_limit='70GB';")
18 + con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb")); con.execute("SET threads TO 24; SET memory_limit='" + os.environ.get("WP44_DUCKDB_MEM", "60GB") + "';")
19 19 con.execute("CREATE OR REPLACE TEMP TABLE E AS SELECT id, publication_year AS y, type, primary_field_id FROM corpus_works WHERE eligible")
20 20 con.execute("""CREATE OR REPLACE TABLE cit_edges_internal AS
21 21 SELECT r.work_id AS citing_id, r.ref_id AS cited_id, a.y AS citing_year, b.y AS cited_year
modified knowledge_graph/02_category_flows.py +1 −1
@@ -21,7 +21,7 @@ def main():
21 21 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); ap.add_argument("--ontology", required=True); a = ap.parse_args(); R = a.root
22 22 out = os.path.join(R, "results/graph"); os.makedirs(out, exist_ok=True)
23 23 ont = yaml.safe_load(open(a.ontology)); cat_area = {c["id"]: c["area"] for c in ont["categories"]}; cats = [c["id"] for c in ont["categories"]]
24 − con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb")); con.execute("SET threads TO 24; SET memory_limit='70GB';")
24 + con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb")); con.execute("SET threads TO 24; SET memory_limit='" + os.environ.get("WP44_DUCKDB_MEM", "60GB") + "';")
25 25 con.execute(f"CREATE OR REPLACE VIEW work_primary AS SELECT * FROM read_parquet('{R}/data/processed/classification/primary/*.parquet')")
26 26 con.execute(f"CREATE OR REPLACE VIEW work_categories AS SELECT * FROM read_parquet('{R}/data/processed/classification/cats/*.parquet')")
27 27 con.execute("CREATE OR REPLACE TEMP TABLE area_map AS SELECT * FROM (VALUES " + ",".join(f"('{c}','{cat_area[c]}')" for c in cats) + ") t(category_id, area)")
modified manuscript/tables/numbers.tex +98 −93
@@ -1,111 +1,116 @@
1 1 % machine-generated by tables/make_numbers.py — do not edit
2 −\newcommand{\PrismaGenerated}{2026-11-03T21:08:38.910490+00:00}
2 +\newcommand{\PrismaGenerated}{2026-11-03T23:03:34.908225+00:00}
3 3 \newcommand{\PrismaBoundary}{2022-11-30}
4 −\newcommand{\PrismaIdentifiedOpenalexCsTopicTwoZeroOneEightTwoZeroTwoSixPassOne}{15{,}490{,}159}
5 −\newcommand{\PrismaIdentifiedOpenalexTotalRecordsLoaded}{15{,}490{,}159}
4 +\newcommand{\PrismaIdentifiedOpenalexCsTopicTwoZeroOneEightTwoZeroTwoSixPassOne}{16{,}592{,}966}
5 +\newcommand{\PrismaIdentifiedOpenalexTotalRecordsLoaded}{16{,}592{,}966}
6 6 \newcommand{\PrismaIdentifiedDblpTwoZeroOneEightTwoZeroTwoSix}{4{,}347{,}365}
7 7 \newcommand{\PrismaIdentifiedArxivCsTwoZeroOneEightTwoZeroTwoSix}{890{,}852}
8 8 \newcommand{\PrismaIdentifiedAclTwoZeroOneEightTwoZeroTwoSix}{85{,}625}
9 −\newcommand{\PrismaLinkedDblpToOpenalex}{2{,}736{,}903}
10 −\newcommand{\PrismaLinkedArxivToOpenalex}{744{,}680}
11 −\newcommand{\PrismaLinkedAclToOpenalex}{72{,}256}
12 −\newcommand{\PrismaLinkedDblpOnlyNoOpenalexRecord}{1{,}610{,}462}
13 −\newcommand{\PrismaExcludedEOneOutsideWindow}{160}
14 −\newcommand{\PrismaExcludedETwoType}{3{,}961{,}633}
15 −\newcommand{\PrismaExcludedEThreeXpac}{3{,}950{,}876}
16 −\newcommand{\PrismaExcludedEFourNoTitle}{16{,}426}
17 −\newcommand{\PrismaExcludedEFiveDuplicate}{114{,}115}
18 −\newcommand{\PrismaExcludedESixRetracted}{25{,}165}
19 −\newcommand{\PrismaExcludedXpacKeptDblp}{87{,}288}
20 −\newcommand{\PrismaExcludedETwoByTypeDataset}{2{,}775{,}759}
21 −\newcommand{\PrismaExcludedETwoByTypeOther}{522{,}654}
9 +\newcommand{\PrismaLinkedDblpToOpenalex}{3{,}774{,}184}
10 +\newcommand{\PrismaLinkedArxivToOpenalex}{864{,}677}
11 +\newcommand{\PrismaLinkedAclToOpenalex}{73{,}054}
12 +\newcommand{\PrismaLinkedDblpOnlyNoOpenalexRecord}{573{,}181}
13 +\newcommand{\PrismaExcludedEOneOutsideWindow}{59{,}453}
14 +\newcommand{\PrismaExcludedETwoType}{3{,}995{,}235}
15 +\newcommand{\PrismaExcludedEThreeXpac}{3{,}974{,}822}
16 +\newcommand{\PrismaExcludedEFourNoTitle}{16{,}442}
17 +\newcommand{\PrismaExcludedEFiveDuplicate}{120{,}147}
18 +\newcommand{\PrismaExcludedESevenGenericDeposit}{1{,}277{,}584}
19 +\newcommand{\PrismaExcludedESixRetracted}{28{,}587}
20 +\newcommand{\PrismaExcludedXpacKeptDblp}{74{,}771}
21 +\newcommand{\PrismaExcludedETwoByTypeDataset}{2{,}779{,}058}
22 +\newcommand{\PrismaExcludedETwoByTypeOther}{538{,}613}
22 23 \newcommand{\PrismaExcludedETwoByTypeLibguides}{191{,}658}
23 −\newcommand{\PrismaExcludedETwoByTypeSoftware}{177{,}500}
24 −\newcommand{\PrismaExcludedETwoByTypeConferenceAbstract}{66{,}267}
25 −\newcommand{\PrismaExcludedETwoByTypePeerReview}{60{,}176}
26 −\newcommand{\PrismaExcludedETwoByTypeParatext}{39{,}162}
27 −\newcommand{\PrismaExcludedETwoByTypeReferenceEntry}{32{,}247}
28 −\newcommand{\PrismaExcludedETwoByTypeEditorial}{24{,}907}
29 −\newcommand{\PrismaExcludedETwoByTypeSupplementaryMaterials}{18{,}882}
30 −\newcommand{\PrismaExcludedETwoByTypeErratum}{17{,}340}
31 −\newcommand{\PrismaExcludedETwoByTypeStandard}{10{,}113}
32 −\newcommand{\PrismaExcludedETwoByTypeBookReview}{9{,}517}
33 −\newcommand{\PrismaExcludedETwoByTypeRetraction}{8{,}217}
34 −\newcommand{\PrismaExcludedETwoByTypeDataPaper}{5{,}255}
35 −\newcommand{\PrismaExcludedETwoByTypeLetter}{1{,}979}
36 −\newcommand{\PrismaExcludedSequentialEOneOutsideWindow}{160}
37 −\newcommand{\PrismaExcludedSequentialETwoType}{3{,}961{,}589}
38 −\newcommand{\PrismaExcludedSequentialEThreeXpac}{1{,}421{,}692}
39 −\newcommand{\PrismaExcludedSequentialEFourNoTitle}{1{,}188}
40 −\newcommand{\PrismaExcludedSequentialEFiveDuplicate}{106{,}928}
41 −\newcommand{\PrismaAfterExclusions}{9{,}998{,}602}
42 −\newcommand{\PrismaCsCoreEligible}{6{,}403{,}238}
43 −\newcommand{\PrismaCsPenumbraEligibleOtherwise}{3{,}595{,}364}
44 −\newcommand{\PrismaEligibleByPeriodPre}{1{,}367}
45 −\newcommand{\PrismaEligibleByPeriodPost}{3{,}457{,}315}
46 −\newcommand{\PrismaEligibleByPeriodBaseline}{2{,}944{,}556}
47 −\newcommand{\PrismaEligibleByYearTwoZeroOneEight}{561{,}564}
48 −\newcommand{\PrismaEligibleByYearTwoZeroOneNine}{600{,}167}
49 −\newcommand{\PrismaEligibleByYearTwoZeroTwoZero}{612{,}005}
50 −\newcommand{\PrismaEligibleByYearTwoZeroTwoOne}{604{,}393}
51 −\newcommand{\PrismaEligibleByYearTwoZeroTwoTwo}{610{,}670}
52 −\newcommand{\PrismaEligibleByYearTwoZeroTwoThree}{678{,}150}
53 −\newcommand{\PrismaEligibleByYearTwoZeroTwoFour}{785{,}101}
54 −\newcommand{\PrismaEligibleByYearTwoZeroTwoFive}{973{,}274}
55 −\newcommand{\PrismaEligibleByYearTwoZeroTwoSix}{977{,}914}
56 −\newcommand{\PrismaEligibleByTypeArticle}{3{,}192{,}312}
57 −\newcommand{\PrismaEligibleByTypeConferencePaper}{1{,}840{,}499}
58 −\newcommand{\PrismaEligibleByTypePreprint}{624{,}265}
59 −\newcommand{\PrismaEligibleByTypeBookChapter}{335{,}735}
60 −\newcommand{\PrismaEligibleByTypeDissertation}{258{,}440}
61 −\newcommand{\PrismaEligibleByTypeReport}{54{,}120}
62 −\newcommand{\PrismaEligibleByTypeBook}{52{,}139}
63 −\newcommand{\PrismaEligibleByTypeReview}{25{,}251}
64 −\newcommand{\PrismaEligibleByTypeOther}{17{,}387}
65 −\newcommand{\PrismaEligibleByTypeSoftwarePaper}{2{,}350}
66 −\newcommand{\PrismaEligibleByTypeBookReview}{615}
67 −\newcommand{\PrismaEligibleByTypeDataPaper}{125}
68 −\newcommand{\PrismaEligibleMembershipInDblp}{2{,}640{,}780}
69 −\newcommand{\PrismaEligibleMembershipCsPrimary}{5{,}619{,}140}
70 −\newcommand{\PrismaEligibleMembershipArxivLinked}{709{,}460}
71 −\newcommand{\PrismaEligibleMembershipInAcl}{70{,}078}
72 −\newcommand{\PrismaEligibleMembershipHasAbstract}{4{,}867{,}840}
73 −\newcommand{\PrismaEligibleMembershipHasReferences}{3{,}715{,}037}
74 −\newcommand{\PrismaEligibleMembershipRetractedFlagged}{14{,}412}
75 −\newcommand{\PrismaStrictCoreDblpAndCsPrimary}{1{,}868{,}188}
76 −\newcommand{\LinkCreated}{2026-11-03T20:50:56.182276+00:00}
77 −\newcommand{\LinkOaCsRecords}{15{,}490{,}159}
78 −\newcommand{\LinkOaCsDistinctIds}{15{,}490{,}159}
24 +\newcommand{\PrismaExcludedETwoByTypeSoftware}{177{,}515}
25 +\newcommand{\PrismaExcludedETwoByTypeConferenceAbstract}{68{,}101}
26 +\newcommand{\PrismaExcludedETwoByTypePeerReview}{60{,}180}
27 +\newcommand{\PrismaExcludedETwoByTypeParatext}{41{,}468}
28 +\newcommand{\PrismaExcludedETwoByTypeReferenceEntry}{33{,}290}
29 +\newcommand{\PrismaExcludedETwoByTypeEditorial}{29{,}543}
30 +\newcommand{\PrismaExcludedETwoByTypeErratum}{20{,}850}
31 +\newcommand{\PrismaExcludedETwoByTypeSupplementaryMaterials}{18{,}946}
32 +\newcommand{\PrismaExcludedETwoByTypeStandard}{10{,}125}
33 +\newcommand{\PrismaExcludedETwoByTypeBookReview}{9{,}524}
34 +\newcommand{\PrismaExcludedETwoByTypeRetraction}{8{,}924}
35 +\newcommand{\PrismaExcludedETwoByTypeDataPaper}{5{,}308}
36 +\newcommand{\PrismaExcludedETwoByTypeLetter}{2{,}132}
37 +\newcommand{\PrismaExcludedSequentialEOneOutsideWindow}{59{,}453}
38 +\newcommand{\PrismaExcludedSequentialETwoType}{3{,}994{,}472}
39 +\newcommand{\PrismaExcludedSequentialEThreeXpac}{1{,}430{,}851}
40 +\newcommand{\PrismaExcludedSequentialEFourNoTitle}{1{,}193}
41 +\newcommand{\PrismaExcludedSequentialEFiveDuplicate}{114{,}421}
42 +\newcommand{\PrismaExcludedSequentialESevenGenericDeposit}{473{,}549}
43 +\newcommand{\PrismaAfterExclusions}{10{,}519{,}027}
44 +\newcommand{\PrismaVersionRelationshipArxivId}{13{,}175}
45 +\newcommand{\PrismaVersionRelationshipTitleYear}{84{,}776}
46 +\newcommand{\PrismaCsCoreEligible}{7{,}110{,}289}
47 +\newcommand{\PrismaCsPenumbraEligibleOtherwise}{3{,}402{,}475}
48 +\newcommand{\PrismaEligibleByPeriodPre}{1{,}864}
49 +\newcommand{\PrismaEligibleByPeriodPost}{3{,}704{,}028}
50 +\newcommand{\PrismaEligibleByPeriodBaseline}{3{,}404{,}397}
51 +\newcommand{\PrismaEligibleByYearTwoZeroOneEight}{642{,}661}
52 +\newcommand{\PrismaEligibleByYearTwoZeroOneNine}{695{,}209}
53 +\newcommand{\PrismaEligibleByYearTwoZeroTwoZero}{708{,}854}
54 +\newcommand{\PrismaEligibleByYearTwoZeroTwoOne}{710{,}953}
55 +\newcommand{\PrismaEligibleByYearTwoZeroTwoTwo}{699{,}308}
56 +\newcommand{\PrismaEligibleByYearTwoZeroTwoThree}{784{,}990}
57 +\newcommand{\PrismaEligibleByYearTwoZeroTwoFour}{901{,}406}
58 +\newcommand{\PrismaEligibleByYearTwoZeroTwoFive}{1{,}065{,}569}
59 +\newcommand{\PrismaEligibleByYearTwoZeroTwoSix}{901{,}339}
60 +\newcommand{\PrismaEligibleByTypeArticle}{3{,}656{,}399}
61 +\newcommand{\PrismaEligibleByTypeConferencePaper}{2{,}138{,}761}
62 +\newcommand{\PrismaEligibleByTypePreprint}{618{,}946}
63 +\newcommand{\PrismaEligibleByTypeBookChapter}{343{,}079}
64 +\newcommand{\PrismaEligibleByTypeDissertation}{237{,}332}
65 +\newcommand{\PrismaEligibleByTypeBook}{47{,}825}
66 +\newcommand{\PrismaEligibleByTypeReport}{33{,}064}
67 +\newcommand{\PrismaEligibleByTypeReview}{28{,}323}
68 +\newcommand{\PrismaEligibleByTypeSoftwarePaper}{3{,}542}
69 +\newcommand{\PrismaEligibleByTypeOther}{1{,}661}
70 +\newcommand{\PrismaEligibleByTypeBookReview}{1{,}197}
71 +\newcommand{\PrismaEligibleByTypeDataPaper}{160}
72 +\newcommand{\PrismaEligibleMembershipInDblp}{3{,}608{,}016}
73 +\newcommand{\PrismaEligibleMembershipCsPrimary}{5{,}322{,}535}
74 +\newcommand{\PrismaEligibleMembershipArxivLinked}{802{,}308}
75 +\newcommand{\PrismaEligibleMembershipInAcl}{70{,}033}
76 +\newcommand{\PrismaEligibleMembershipHasAbstract}{5{,}385{,}235}
77 +\newcommand{\PrismaEligibleMembershipHasReferences}{4{,}614{,}417}
78 +\newcommand{\PrismaEligibleMembershipRetractedFlagged}{17{,}094}
79 +\newcommand{\PrismaEligibleDatePrecisionMonth}{5{,}703{,}718}
80 +\newcommand{\PrismaEligibleDatePrecisionYear}{1{,}406{,}571}
81 +\newcommand{\PrismaStrictCoreDblpAndCsPrimary}{1{,}832{,}623}
82 +\newcommand{\LinkCreated}{2026-11-03T23:03:10.160666+00:00}
83 +\newcommand{\LinkOaCsRecords}{16{,}592{,}966}
84 +\newcommand{\LinkOaCsDistinctIds}{16{,}592{,}966}
79 85 \newcommand{\LinkDblpTwoZeroOneEightTwoZeroTwoSix}{4{,}347{,}365}
80 86 \newcommand{\LinkDblpWithDoi}{4{,}052{,}326}
81 87 \newcommand{\LinkDblpWithArxiv}{0}
82 −\newcommand{\LinkLinkDblpOaTitleYear}{179{,}990}
83 −\newcommand{\LinkLinkDblpOaDoi}{2{,}562{,}431}
84 −\newcommand{\LinkDblpLinkedDistinct}{2{,}736{,}903}
85 −\newcommand{\LinkDblpUnlinked}{1{,}610{,}462}
86 −\newcommand{\LinkDblpUnlinkedByTypePhdthesis}{26{,}841}
87 −\newcommand{\LinkDblpUnlinkedByTypeIncollection}{3{,}362}
88 −\newcommand{\LinkDblpUnlinkedByTypeInproceedings}{477{,}235}
89 −\newcommand{\LinkDblpUnlinkedByTypeBook}{1{,}398}
90 −\newcommand{\LinkDblpUnlinkedByTypeArticle}{1{,}084{,}640}
91 −\newcommand{\LinkDblpUnlinkedByTypeData}{16{,}986}
92 −\newcommand{\LinkDblpUnlinkedWithDoiInSlim}{1{,}077{,}407}
88 +\newcommand{\LinkLinkDblpOaTitleYear}{141{,}062}
89 +\newcommand{\LinkLinkDblpOaDoi}{3{,}639{,}848}
90 +\newcommand{\LinkDblpLinkedDistinct}{3{,}774{,}184}
91 +\newcommand{\LinkDblpUnlinked}{573{,}181}
92 +\newcommand{\LinkDblpUnlinkedByTypeIncollection}{418}
93 +\newcommand{\LinkDblpUnlinkedByTypeBook}{629}
94 +\newcommand{\LinkDblpUnlinkedByTypePhdthesis}{30{,}292}
95 +\newcommand{\LinkDblpUnlinkedByTypeInproceedings}{159{,}308}
96 +\newcommand{\LinkDblpUnlinkedByTypeArticle}{367{,}850}
97 +\newcommand{\LinkDblpUnlinkedByTypeData}{14{,}684}
98 +\newcommand{\LinkDblpUnlinkedWithDoiInSlim}{1{,}605}
93 99 \newcommand{\LinkArxivCsRecords}{915{,}115}
94 100 \newcommand{\LinkArxivCreatedTwoZeroOneEightTwoZeroTwoSix}{890{,}852}
95 −\newcommand{\LinkLinkArxivOaArxivId}{792{,}672}
96 −\newcommand{\LinkLinkArxivOaDoi}{520}
97 −\newcommand{\LinkLinkArxivOaTitleYear}{2{,}143}
98 −\newcommand{\LinkArxivLinkedDistinct}{744{,}680}
101 +\newcommand{\LinkLinkArxivOaTitleYear}{1{,}768}
102 +\newcommand{\LinkLinkArxivOaDoi}{783}
103 +\newcommand{\LinkLinkArxivOaArxivId}{914{,}406}
104 +\newcommand{\LinkArxivLinkedDistinct}{864{,}677}
99 105 \newcommand{\LinkAclRecords}{135{,}558}
100 106 \newcommand{\LinkAclTwoZeroOneEightTwoZeroTwoSix}{85{,}625}
101 −\newcommand{\LinkLinkAclOaDoi}{57{,}474}
102 −\newcommand{\LinkLinkAclOaTitleYear}{14{,}904}
103 −\newcommand{\LinkAclLinkedDistinctTwoZeroOneEightTwoZeroTwoSix}{72{,}223}
104 −\newcommand{\LinkNeededOaIdsSecondPassDblpDoi}{1{,}077{,}406}
105 −\newcommand{\LinkNeededOaIdsSecondPassArxivDoi}{39{,}200}
107 +\newcommand{\LinkLinkAclOaDoi}{58{,}174}
108 +\newcommand{\LinkLinkAclOaTitleYear}{15{,}004}
109 +\newcommand{\LinkAclLinkedDistinctTwoZeroOneEightTwoZeroTwoSix}{72{,}949}
110 +\newcommand{\LinkNeededOaIdsSecondPassDblpDoi}{1{,}605}
106 111 \newcommand{\LinkSlimTotal}{464{,}765{,}485}
107 112 \newcommand{\LinkSlimCsAnyTwoZeroOneEightTwoZeroTwoSix}{15{,}490{,}159}
108 −\newcommand{\LinkSeconds}{10}
113 +\newcommand{\LinkSeconds}{12}
109 114 \newcommand{\DblpParsedRecords}{8{,}840{,}866}
110 115 \newcommand{\DblpTypeIncollection}{71{,}247}
111 116 \newcommand{\DblpTypeBook}{21{,}648}
modified systematic_review/define_corpus.py +1 −1
@@ -18,7 +18,7 @@ WINDOW_START, BOUNDARY, WINDOW_END = "2018-01-01", "2022-11-30", "2026-11-02"
18 18
19 19 def main():
20 20 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); a = ap.parse_args(); R = a.root
21 − con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb")); con.execute("SET threads TO 24; SET memory_limit='60GB';")
21 + con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb")); con.execute("SET threads TO 24; SET memory_limit='" + os.environ.get("WP44_DUCKDB_MEM", "60GB") + "';")
22 22 t0 = time.time(); q = lambda s: con.execute(s).fetchall(); n = lambda s: q(s)[0][0]
23 23 log = lambda m: print(f"[{time.time()-t0:6.0f}s] {m}", flush=True)
24 24 # ---- S0/S1: base table with membership flags ----
modified taxonomy/classify.py +1 −1
@@ -95,7 +95,7 @@ def process_chunk(args):
95 95
96 96
97 97 def prepare(root, db, out_dir, chunk_rows):
98 − con = duckdb.connect(db); con.execute("SET threads TO 24; SET memory_limit='60GB';")
98 + con = duckdb.connect(db); con.execute("SET threads TO 24; SET memory_limit='" + os.environ.get("WP44_DUCKDB_MEM", "60GB") + "';")
99 99 os.makedirs(os.path.join(out_dir, "input"), exist_ok=True)
100 100 con.execute("""CREATE OR REPLACE TABLE cls_input AS
101 101 SELECT w.id, w.publication_year AS year, w.title, a.abstract,
102 102
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere