1#!/usr/bin/env python32"""Build and validate the WP44 computer-science ontology.34Reads the mapping data modules in taxonomy/src/ (area_*.py, areas.py, unmapped_cs.py), validates them against the5OpenAlex topic export and the DBLP stream export, and writes:6 taxonomy/ontology.yaml (schema of agents/briefs/AGENT_D_TAXONOMY.md)7 taxonomy/topic_map.csv (one row per (topic_id, category_id, weight))8 taxonomy/ONTOLOGY.md (generated section between <!-- BEGIN GENERATED --> / <!-- END GENERATED --> markers;9 hand-written prose outside the markers is preserved)10Run: /opt/homebrew/bin/python3 taxonomy/build_ontology.py [--no-write]11Exit status is non-zero if any validation fails.12"""13from __future__ import annotations1415import csv16import datetime as dt17import glob18import importlib.util19import os20import re21import sys22from collections import Counter, defaultdict2324HERE = os.path.dirname(os.path.abspath(__file__))25SRC = os.path.join(HERE, "src")26TOPICS_CSV = os.path.join(HERE, "openalex_topics_2026-10-14.csv")27STREAMS_CSV = os.path.join(HERE, "dblp_streams_2018_2026.csv")28OUT_YAML = os.path.join(HERE, "ontology.yaml")29OUT_MAP = os.path.join(HERE, "topic_map.csv")30OUT_MD = os.path.join(HERE, "ONTOLOGY.md")31VERSION, CREATED = "1.0", "2026-11-03"32CS_FIELD = "17"33TOP_STREAMS = 60034MARK_BEGIN, MARK_END = "<!-- BEGIN GENERATED -->", "<!-- END GENERATED -->"35FIELDS = {"title", "abstract", "both"}36LIST_KEYS = ("topics", "subfields", "arxiv", "streams", "keywords", "exclusions", "related", "parent_of", "child_of")3738errors: list[str] = []394041def err(msg: str) -> None:42 errors.append(msg)434445def load_module(path: str):46 name = os.path.splitext(os.path.basename(path))[0]47 spec = importlib.util.spec_from_file_location(name, path)48 mod = importlib.util.module_from_spec(spec)49 spec.loader.exec_module(mod) # type: ignore[union-attr]50 return mod515253def wt_ok(w) -> bool:54 return isinstance(w, (int, float)) and 0 < float(w) <= 1555657def main(write: bool = True) -> int:58 topics = {r["topic_id"]: r for r in csv.DictReader(open(TOPICS_CSV, encoding="utf-8"))}59 subfields = {int(r["subfield_id"]): r["subfield"] for r in topics.values()}60 streams = {r["stream"]: r for r in csv.DictReader(open(STREAMS_CSV, encoding="utf-8"))}61 areas_mod = load_module(os.path.join(SRC, "areas.py"))62 unmapped_mod = load_module(os.path.join(SRC, "unmapped_cs.py"))63 AREAS = {a[0]: {"id": a[0], "name": a[1], "description": a[2]} for a in areas_mod.AREAS}64 ARXIV = areas_mod.ARXIV_CATEGORIES6566 cats: list[dict] = []67 for path in sorted(glob.glob(os.path.join(SRC, "area_*.py"))):68 mod = load_module(path)69 for c in mod.CATEGORIES:70 c["_module"] = os.path.basename(path)71 cats.append(c)72 cats.sort(key=lambda c: c["number"])73 ids = [c["id"] for c in cats]74 idset = set(ids)7576 # ---- structural validation -------------------------------------------------------------------------------------77 if len(cats) != 40:78 err(f"expected 40 categories, found {len(cats)}")79 if len(idset) != len(ids):80 err(f"duplicate category ids: {[i for i, n in Counter(ids).items() if n > 1]}")81 if sorted(c["number"] for c in cats) != list(range(1, 41)):82 err("category numbers are not exactly 1..40")83 for c in cats:84 cid = c["id"]85 if not re.fullmatch(r"[a-z][a-z0-9_]*", cid):86 err(f"{cid}: id is not snake_case")87 if c["area"] not in AREAS:88 err(f"{cid}: unknown area {c['area']}")89 if not isinstance(c.get("definition"), str) or len(c["definition"]) < 60:90 err(f"{cid}: definition missing or too short")91 if not c.get("provenance"):92 err(f"{cid}: provenance missing")93 for k in LIST_KEYS:94 if not isinstance(c.get(k), list):95 err(f"{cid}: field {k} must be a list")96 # duplicates inside signal lists97 for k, keyfn in (("topics", lambda t: t[0]), ("subfields", lambda t: t[0]), ("arxiv", lambda t: t[0]),98 ("streams", lambda t: t[0]), ("keywords", lambda t: t[0])):99 keys = [keyfn(t) for t in c[k]]100 dup = [x for x, n in Counter(keys).items() if n > 1]101 if dup:102 err(f"{cid}: duplicate entries in {k}: {dup}")103 for tid, w, note in c["topics"]:104 if tid not in topics:105 err(f"{cid}: unknown OpenAlex topic id {tid}")106 if not wt_ok(w):107 err(f"{cid}: topic {tid} weight {w} not in (0,1]")108 if not note:109 err(f"{cid}: topic {tid} has no note")110 for sid, w in c["subfields"]:111 if sid not in subfields:112 err(f"{cid}: unknown subfield id {sid}")113 if not wt_ok(w):114 err(f"{cid}: subfield {sid} weight {w} not in (0,1]")115 for a, w in c["arxiv"]:116 if a not in ARXIV:117 err(f"{cid}: arXiv category {a} not in admitted list")118 if not wt_ok(w):119 err(f"{cid}: arXiv {a} weight {w} not in (0,1]")120 for s, w in c["streams"]:121 if s not in streams:122 err(f"{cid}: unknown DBLP stream {s}")123 if not wt_ok(w):124 err(f"{cid}: stream {s} weight {w} not in (0,1]")125 for pat, w, field in c["keywords"]:126 try:127 re.compile(pat)128 except re.error as e:129 err(f"{cid}: keyword regex does not compile: {pat[:60]}... ({e})")130 if not wt_ok(w):131 err(f"{cid}: keyword weight {w} not in (0,1]: {pat[:40]}")132 if field not in FIELDS:133 err(f"{cid}: keyword field {field!r} invalid")134 for pat in c["exclusions"]:135 try:136 re.compile(pat)137 except re.error as e:138 err(f"{cid}: exclusion regex does not compile: {pat[:60]}... ({e})")139 for k in ("related", "parent_of", "child_of"):140 for other in c[k]:141 if other not in idset:142 err(f"{cid}: {k} references unknown category {other}")143 if other == cid:144 err(f"{cid}: {k} references itself")145 if len(set(c["related"])) != len(c["related"]):146 err(f"{cid}: duplicate related entries")147 # parent/child symmetry148 by_id = {c["id"]: c for c in cats}149 for c in cats:150 for ch in c["parent_of"]:151 if ch in by_id and c["id"] not in by_id[ch]["child_of"]:152 err(f"{c['id']} parent_of {ch} but {ch} does not declare child_of {c['id']}")153 for p in c["child_of"]:154 if p in by_id and c["id"] not in by_id[p]["parent_of"]:155 err(f"{c['id']} child_of {p} but {p} does not declare parent_of {c['id']}")156 # keyword sufficiency for the categories the brief singles out157 must_keyword = {"llm", "foundation_models", "multimodal_ai", "generative_ai", "ai_agents", "ai_safety",158 "federated_learning", "sustainable_computing", "autonomous_scientific_discovery",159 "privacy_preserving_computation", "blockchain", "quantum_computing", "edge_computing"}160 kw_sufficient = {c["id"]: any(float(w) >= 1.0 for _, w, _ in c["keywords"]) for c in cats}161 for cid in must_keyword:162 if cid in by_id and not kw_sufficient[cid]:163 err(f"{cid}: must have at least one keyword pattern of weight 1.0 (threshold reachable alone)")164165 # ---- coverage statistics ---------------------------------------------------------------------------------------166 topic_cats: dict[str, list[tuple[str, float]]] = defaultdict(list)167 for c in cats:168 for tid, w, _ in c["topics"]:169 topic_cats[tid].append((c["id"], float(w)))170 cs_topics = {t for t, r in topics.items() if r["field_id"] == CS_FIELD}171 cs_mapped = sorted(t for t in cs_topics if t in topic_cats)172 cs_unmapped = sorted(t for t in cs_topics if t not in topic_cats)173 reasons = unmapped_mod.UNMAPPED_REASONS174 for t in cs_unmapped:175 if t not in reasons:176 err(f"unmapped CS topic {t} ({topics[t]['display_name']}, subfield {topics[t]['subfield_id']}, "177 f"{topics[t]['works_count']} works) has no reason in src/unmapped_cs.py")178 for t in reasons:179 if t not in topics:180 err(f"unmapped_cs.py: unknown topic id {t}")181 elif t in topic_cats:182 err(f"unmapped_cs.py: topic {t} is listed as unmapped but is mapped to {[c for c, _ in topic_cats[t]]}")183 noncs_mapped = sorted(t for t in topic_cats if topics[t]["field_id"] != CS_FIELD)184 noncs_by_field = Counter(f"{topics[t]['field_id']} {topics[t]['field']}" for t in noncs_mapped)185186 stream_cats: dict[str, list[tuple[str, float]]] = defaultdict(list)187 for c in cats:188 for s, w in c["streams"]:189 stream_cats[s].append((c["id"], float(w)))190 ranked = sorted(streams.values(), key=lambda r: -int(r["n"]))191 top = [r for r in ranked[:TOP_STREAMS] if r["stream"] != "journals/corr"]192 top_mapped = [r for r in top if r["stream"] in stream_cats]193 top_unmapped = [r for r in top if r["stream"] not in stream_cats]194 total_recs = sum(int(r["n"]) for r in ranked if r["stream"] != "journals/corr")195 mapped_recs = sum(int(streams[s]["n"]) for s in stream_cats if s in streams and s != "journals/corr")196197 if errors:198 print("VALIDATION FAILED")199 for e in errors:200 print(" -", e)201 return 1202203 # ---- outputs ---------------------------------------------------------------------------------------------------204 import yaml # imported late so validation errors are reported even without pyyaml205206 def cat_yaml(c: dict) -> dict:207 return {208 "id": c["id"], "number": c["number"], "name": c["name"], "area": c["area"], "definition": c["definition"],209 "openalex_topics": [{"id": t, "weight": float(w), "note": n} for t, w, n in c["topics"]],210 "openalex_subfields": [{"id": int(s), "weight": float(w)} for s, w in c["subfields"]],211 "arxiv_categories": [{"id": a, "weight": float(w)} for a, w in c["arxiv"]],212 "dblp_streams": [{"key": s, "weight": float(w)} for s, w in c["streams"]],213 "keywords": [{"pattern": p, "weight": float(w), "field": f} for p, w, f in c["keywords"]],214 "exclusions": list(c["exclusions"]),215 "related": list(c["related"]), "parent_of": list(c["parent_of"]), "child_of": list(c["child_of"]),216 "provenance": c["provenance"],217 }218219 area_leaves = defaultdict(list)220 for c in cats:221 area_leaves[c["area"]].append(c["id"])222 doc = {223 "version": VERSION, "created": CREATED,224 "sources": {"openalex_topics": os.path.basename(TOPICS_CSV), "dblp_streams": os.path.basename(STREAMS_CSV),225 "generator": "taxonomy/build_ontology.py", "data_modules": "taxonomy/src/*.py"},226 "areas": [{"id": a["id"], "name": a["name"], "description": a["description"], "leaves": area_leaves[a["id"]]}227 for a in AREAS.values()],228 "categories": [cat_yaml(c) for c in cats],229 "assignment_rule": {230 "score": "sum of weights of matched signals, each signal type capped at 1.0, then sum over signal types",231 "signal_types": list(areas_mod.SIGNAL_TYPES),232 "threshold": 1.0,233 "primary": "argmax of score; ties broken by the more specific category (child before parent), then by lower number",234 "exclusions": "if any exclusion regex matches the concatenated title+abstract, the category is vetoed for that work",235 "keyword_field": "title patterns are matched on the title only; abstract on the abstract only; both on title+abstract",236 "notes": "OpenAlex topic weights assume the work's primary topic (or any of its up-to-three topics; the classifier "237 "takes the sum of matched topic weights capped at 1.0 before capping). Weights below 1.0 are deliberate: such signals must be "238 "corroborated by another signal type to reach the threshold.",239 },240 }241 if write:242 with open(OUT_YAML, "w", encoding="utf-8") as fh:243 yaml.safe_dump(doc, fh, allow_unicode=True, sort_keys=False, width=1000)244 # round-trip check: every regex survives YAML serialisation245 back = yaml.safe_load(open(OUT_YAML, encoding="utf-8"))246 for c0, c1 in zip(cats, back["categories"]):247 for (p, _, _), k in zip(c0["keywords"], c1["keywords"]):248 assert p == k["pattern"], f"YAML round-trip altered a regex in {c0['id']}"249 re.compile(k["pattern"])250 for p, q in zip(c0["exclusions"], c1["exclusions"]):251 assert p == q252 re.compile(q)253 with open(OUT_MAP, "w", encoding="utf-8", newline="") as fh:254 w = csv.writer(fh)255 w.writerow(["topic_id", "category_id", "weight", "topic_name", "field_id", "field", "subfield_id", "subfield",256 "works_count", "note"])257 for c in cats:258 for tid, wt, note in sorted(c["topics"], key=lambda t: (-float(t[1]), t[0])):259 r = topics[tid]260 w.writerow([tid, c["id"], f"{float(wt):.2f}", r["display_name"], r["field_id"], r["field"],261 r["subfield_id"], r["subfield"], r["works_count"], note])262263 # ---- generated markdown ----------------------------------------------------------------------------------------264 L: list[str] = []265 L.append(f"_Generated by `taxonomy/build_ontology.py` on {dt.date.today().isoformat()} from "266 f"`{os.path.basename(TOPICS_CSV)}` ({len(topics):,} topics) and `{os.path.basename(STREAMS_CSV)}` "267 f"({len(streams):,} streams). Do not edit between the markers._\n")268 L.append("## Areas and leaves\n")269 L.append("| Area | Name | Leaves |")270 L.append("|---|---|---|")271 for a in AREAS.values():272 L.append(f"| `{a['id']}` | {a['name']} | {', '.join('`'+x+'`' for x in area_leaves[a['id']])} |")273 L.append("\n## The 40 categories\n")274 for c in cats:275 L.append(f"**{c['number']}. {c['name']}** (`{c['id']}`, area `{c['area']}`) ")276 L.append(f"{c['definition']} ")277 edges = []278 if c["child_of"]:279 edges.append("child of " + ", ".join(f"`{x}`" for x in c["child_of"]))280 if c["parent_of"]:281 edges.append("parent of " + ", ".join(f"`{x}`" for x in c["parent_of"]))282 if c["related"]:283 edges.append("related to " + ", ".join(f"`{x}`" for x in c["related"]))284 L.append("_" + "; ".join(edges) + "._\n")285 L.append("## Hierarchy (parent → child edges)\n")286 for c in cats:287 for ch in c["parent_of"]:288 L.append(f"- `{c['id']}` → `{ch}`")289 L.append("\n## Signal counts per category\n")290 L.append("| # | id | topics | subfields | arXiv | DBLP streams | keywords | exclusions | keyword alone reaches threshold |")291 L.append("|---|---|---|---|---|---|---|---|---|")292 for c in cats:293 L.append(f"| {c['number']} | `{c['id']}` | {len(c['topics'])} | {len(c['subfields'])} | {len(c['arxiv'])} | "294 f"{len(c['streams'])} | {len(c['keywords'])} | {len(c['exclusions'])} | {'yes' if kw_sufficient[c['id']] else 'no'} |")295 L.append(f"\nTotals: {sum(len(c['topics']) for c in cats)} topic→category weights over {len(topic_cats)} distinct topics; "296 f"{sum(len(c['streams']) for c in cats)} stream→category weights over {len(stream_cats)} distinct streams; "297 f"{sum(len(c['keywords']) for c in cats)} keyword patterns; {sum(len(c['exclusions']) for c in cats)} exclusion patterns.\n")298 L.append("## OpenAlex coverage\n")299 L.append(f"- Field 17 (Computer Science): **{len(cs_mapped)} of {len(cs_topics)} topics mapped** "300 f"({100*len(cs_mapped)/len(cs_topics):.1f} %); {len(cs_unmapped)} left unmapped with an explicit reason below.")301 L.append(f"- Topics mapped from other fields: {len(noncs_mapped)} — " +302 "; ".join(f"{k}: {v}" for k, v in sorted(noncs_by_field.items())) + ".")303 multi = sum(1 for t, l in topic_cats.items() if len(l) > 1)304 L.append(f"- Topics mapped to more than one category: {multi} of {len(topic_cats)}.")305 L.append("\n### Mapped CS topics per subfield\n")306 L.append("| Subfield | topics | mapped | unmapped |")307 L.append("|---|---|---|---|")308 bysf = defaultdict(lambda: [0, 0])309 for t in cs_topics:310 bysf[(topics[t]["subfield_id"], topics[t]["subfield"])][0 if t in topic_cats else 1] += 1311 for (sid, sname), (m, u) in sorted(bysf.items()):312 L.append(f"| {sid} {sname} | {m+u} | {m} | {u} |")313 L.append("\n### Unmapped CS topics (field 17) with reason\n")314 L.append("| topic | name | subfield | works | reason |")315 L.append("|---|---|---|---|---|")316 for t in sorted(cs_unmapped, key=lambda t: -int(topics[t]["works_count"])):317 r = topics[t]318 L.append(f"| {t} | {r['display_name']} | {r['subfield_id']} | {int(r['works_count']):,} | {reasons[t]} |")319 L.append("\n## DBLP coverage\n")320 L.append(f"- Streams mapped: **{len(stream_cats)}** (any rank); records covered (2018–2026, excluding CoRR): "321 f"{mapped_recs:,} of {total_recs:,} ({100*mapped_recs/total_recs:.1f} %).")322 L.append(f"- Of the {len(top)} largest streams (CoRR excluded), {len(top_mapped)} are mapped "323 f"({100*len(top_mapped)/len(top):.1f} %). The {len(top_unmapped)} unmapped large streams are listed below; they "324 f"are either out of scope (medicine, geoscience, education, management, electronics) or too generic to be a "325 f"category signal (IEEE Access, Sensors, Symmetry).")326 L.append("\n| stream | records | name |")327 L.append("|---|---|---|")328 for r in top_unmapped:329 L.append(f"| `{r['stream']}` | {int(r['n']):,} | {(r['journal'] or r['booktitle'])} |")330 L.append("\n## Topics mapped to several categories (audit list)\n")331 L.append("| topic | name | categories (weight) |")332 L.append("|---|---|---|")333 for t in sorted(topic_cats, key=lambda t: (-len(topic_cats[t]), t)):334 if len(topic_cats[t]) > 1:335 L.append(f"| {t} | {topics[t]['display_name']} | " +336 ", ".join(f"`{c}` ({w:.1f})" for c, w in sorted(topic_cats[t], key=lambda x: -x[1])) + " |")337 generated = "\n".join(L) + "\n"338339 if write:340 if os.path.exists(OUT_MD):341 text = open(OUT_MD, encoding="utf-8").read()342 if MARK_BEGIN in text and MARK_END in text:343 pre = text.split(MARK_BEGIN)[0]344 post = text.split(MARK_END)[1]345 text = pre + MARK_BEGIN + "\n" + generated + MARK_END + post346 else:347 text = text.rstrip() + "\n\n" + MARK_BEGIN + "\n" + generated + MARK_END + "\n"348 else:349 text = ("# WP44 Ontology of Computer-Science Research\n\n(prose to be written)\n\n" + MARK_BEGIN + "\n" +350 generated + MARK_END + "\n")351 open(OUT_MD, "w", encoding="utf-8").write(text)352353 # ---- console summary -------------------------------------------------------------------------------------------354 print(f"OK: 40 categories, {len(topic_cats)} topics, {len(stream_cats)} streams, "355 f"{sum(len(c['keywords']) for c in cats)} keyword patterns")356 print(f"CS topics mapped {len(cs_mapped)}/{len(cs_topics)}; non-CS topics mapped {len(noncs_mapped)}; "357 f"top-{TOP_STREAMS} streams mapped {len(top_mapped)}/{len(top)}; records covered {100*mapped_recs/total_recs:.1f}%")358 print("per category (topics/streams/keywords):")359 for c in cats:360 print(f" {c['number']:2d} {c['id']:32s} {len(c['topics']):3d} {len(c['streams']):3d} {len(c['keywords']):3d}"361 f"{'' if kw_sufficient[c['id']] else ' (no 1.0 keyword)'}")362 return 0363364365if __name__ == "__main__":366 sys.exit(main(write="--no-write" not in sys.argv))367