SPB Git
spboucher.ai

spb/wp44_uqo

Public

WP44: ontology v1.0 (Agent D), classifier shard mode, taxonomy tests

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Nov 3, 2026) parent 9ea4143 Browse files

4 changed files +35 −4

4 changed files
Unified Split
modified agents/REGISTER.md +1 −1
@@ -4,5 +4,5 @@
4 |---|---|---|---|---|---| 4 |---|---|---|---|---|---|
5 | 2026-11-02 | A (director, this session) | CLAUDE.md | RESEARCH_PLAN.md, protocol/PROTOCOL.md, reports/PHASE0_FEASIBILITY.md | PI checkpoint H-0 pending | delivered | 5 | 2026-11-02 | A (director, this session) | CLAUDE.md | RESEARCH_PLAN.md, protocol/PROTOCOL.md, reports/PHASE0_FEASIBILITY.md | PI checkpoint H-0 pending | delivered |
6 | 2026-11-02/03 | B+C (acquisition & data engineering, this session) | RESEARCH_PLAN §2–3 | ingestion/*.py, ledgers on M3U96b, data/metadata/linkage_report.json | N (audit, Phase 8) pending | delivered | 6 | 2026-11-02/03 | B+C (acquisition & data engineering, this session) | RESEARCH_PLAN §2–3 | ingestion/*.py, ledgers on M3U96b, data/metadata/linkage_report.json | N (audit, Phase 8) pending | delivered |
7−| 2026-11-03 | D (taxonomy architect, subagent, 2nd attempt) | agents/briefs/AGENT_D_TAXONOMY.md | taxonomy/ontology.yaml, topic_map.csv, ONTOLOGY.md, src/ | A + K (validation sample, Phase 2) | running | 7+| 2026-11-03 | D (taxonomy architect, subagent, 2nd attempt) | agents/briefs/AGENT_D_TAXONOMY.md | taxonomy/ontology.yaml (v1.0, 40 leaves, 8 areas, 646 topic weights, 771 DBLP streams, 273 regexes, 62 unmapped CS topics listed), topic_map.csv, ONTOLOGY.md, src/ | A (build re-run, classify smoke test 2026-11-03) + validation sample (Phase 2) | delivered 2026-11-03 |
8 | 2026-11-03 | B-lit (reference acquisition, subagent) | agents/briefs/AGENT_LIT_REFERENCES.md | manuscript/references.bib, references_verification.csv, literature_synthesis/REFERENCE_MAP.md | M (compile) + N (re-verify sample) | running | 8 | 2026-11-03 | B-lit (reference acquisition, subagent) | agents/briefs/AGENT_LIT_REFERENCES.md | manuscript/references.bib, references_verification.csv, literature_synthesis/REFERENCE_MAP.md | M (compile) + N (re-verify sample) | running |
modified taxonomy/classify.py +13 −2
@@ -28,7 +28,7 @@ def load_ontology(path):
28 kws.append((re.compile(k["pattern"]), float(k["weight"]), k.get("field", "both"))) 28 kws.append((re.compile(k["pattern"]), float(k["weight"]), k.get("field", "both")))
29 exc = [re.compile(p if isinstance(p, str) else p["pattern"]) for p in (c.get("exclusions") or [])] 29 exc = [re.compile(p if isinstance(p, str) else p["pattern"]) for p in (c.get("exclusions") or [])]
30 comp.append(dict( 30 comp.append(dict(
31− id=c["id"], number=int(c["number"]), parent=c.get("child_of"), 31+ id=c["id"], number=int(c["number"]), parent=tuple(c["child_of"]) if isinstance(c.get("child_of"), list) else ((c["child_of"],) if c.get("child_of") else ()),
32 topics={str(t["id"]).lstrip("T"): float(t["weight"]) for t in (c.get("openalex_topics") or [])}, 32 topics={str(t["id"]).lstrip("T"): float(t["weight"]) for t in (c.get("openalex_topics") or [])},
33 subfields={str(t["id"]): float(t["weight"]) for t in (c.get("openalex_subfields") or [])}, 33 subfields={str(t["id"]): float(t["weight"]) for t in (c.get("openalex_subfields") or [])},
34 arxiv={t["id"]: float(t["weight"]) for t in (c.get("arxiv_categories") or [])}, 34 arxiv={t["id"]: float(t["weight"]) for t in (c.get("arxiv_categories") or [])},
@@ -81,7 +81,7 @@ def process_chunk(args):
81 for (cid, num, parent, s, st, ss, sa, sd, sk) in res: 81 for (cid, num, parent, s, st, ss, sa, sd, sk) in res:
82 cw.append(t["id"][i]); cc.append(cid); cs.append(s); c1.append(st); c2.append(ss); c3.append(sa); c4.append(sd); c5.append(sk) 82 cw.append(t["id"][i]); cc.append(cid); cs.append(s); c1.append(st); c2.append(ss); c3.append(sa); c4.append(sd); c5.append(sk)
83 if res: 83 if res:
84− parents = {r[2] for r in res if r[2]} 84+ parents = set().union(*[set(r[2]) for r in res])
85 # primary: highest score; tie → child before parent (a category that is someone's parent loses), then lower number 85 # primary: highest score; tie → child before parent (a category that is someone's parent loses), then lower number
86 best = sorted(res, key=lambda r: (-r[3], 1 if r[0] in parents else 0, r[1]))[0] 86 best = sorted(res, key=lambda r: (-r[3], 1 if r[0] in parents else 0, r[1]))[0]
87 pw.append(t["id"][i]); pc.append(best[0]); pn.append(len(res)); ps.append(best[3]) 87 pw.append(t["id"][i]); pc.append(best[0]); pn.append(len(res)); ps.append(best[3])
@@ -113,9 +113,20 @@ def main():
113 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); ap.add_argument("--ontology", required=True) 113 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); ap.add_argument("--ontology", required=True)
114 ap.add_argument("--prepare", action="store_true"); ap.add_argument("--run", action="store_true"); ap.add_argument("--workers", type=int, default=20) 114 ap.add_argument("--prepare", action="store_true"); ap.add_argument("--run", action="store_true"); ap.add_argument("--workers", type=int, default=20)
115 ap.add_argument("--chunk-rows", type=int, default=100000); ap.add_argument("--limit", type=int, default=0) 115 ap.add_argument("--chunk-rows", type=int, default=100000); ap.add_argument("--limit", type=int, default=0)
116+ ap.add_argument("--shard", default=None, help="i/N: process only chunks with index %% N == i in this process (no pool; used by run_classify_shards.sh)")
116 a = ap.parse_args(); out_dir = os.path.join(a.root, "data/processed/classification"); db = os.path.join(a.root, "data/processed/wp44.duckdb") 117 a = ap.parse_args(); out_dir = os.path.join(a.root, "data/processed/classification"); db = os.path.join(a.root, "data/processed/wp44.duckdb")
117 if a.prepare: 118 if a.prepare:
118 prepare(a.root, db, out_dir, a.chunk_rows) 119 prepare(a.root, db, out_dir, a.chunk_rows)
120+ if a.run and a.shard:
121+ i, N = map(int, a.shard.split("/")); _init(a.ontology)
122+ files = sorted(glob.glob(os.path.join(out_dir, "input", "chunk=*", "*.parquet")), key=lambda f: int(os.path.basename(os.path.dirname(f)).split("=")[1]))
123+ t0 = time.time(); tot = cl = 0
124+ for k, f in enumerate(files):
125+ if k % N != i:
126+ continue
127+ name, n, c = process_chunk_named((f, out_dir, os.path.basename(os.path.dirname(f)))); tot += n; cl += c
128+ print(f"[{time.time()-t0:6.0f}s] shard {i}/{N} {name}: {n} works, {c} classified | {tot} / {cl}", flush=True)
129+ return
119 if a.run: 130 if a.run:
120 files = sorted(glob.glob(os.path.join(out_dir, "input", "chunk=*", "*.parquet"))) 131 files = sorted(glob.glob(os.path.join(out_dir, "input", "chunk=*", "*.parquet")))
121 files = [(f, out_dir) for f in files] 132 files = [(f, out_dir) for f in files]
added taxonomy/run_classify_shards.sh +20 −0
@@ -0,0 +1,20 @@
1+#!/bin/sh
2+# Classify the corpus with N independent processes (macOS multiprocessing pool hangs with pyarrow in spawned workers).
3+# Usage: sh taxonomy/run_classify_shards.sh [N] — waits for the pass-2 chain (CHAIN_DONE) before preparing the input.
4+cd ~/wp44_uqo; . .venv/bin/activate; ulimit -n 65536; N=${1:-20}
5+while ! grep -q CHAIN_DONE logs/define_corpus_v2.log 2>/dev/null; do sleep 60; done
6+rm -rf data/processed/classification; mkdir -p logs/classify
7+python taxonomy/classify.py --root ~/wp44_uqo --ontology taxonomy/ontology.yaml --prepare > logs/classify/prepare.log 2>&1
8+i=0; while [ $i -lt $N ]; do
9+ nohup python taxonomy/classify.py --root ~/wp44_uqo --ontology taxonomy/ontology.yaml --run --shard $i/$N > logs/classify/shard_$i.log 2>&1 &
10+ i=$((i+1)); done
11+wait
12+python - <<EOF2
13+import duckdb, json, datetime as dt, yaml
14+con = duckdb.connect(); o = yaml.safe_load(open("taxonomy/ontology.yaml"))
15+n = con.sql("select count(distinct work_id) from read_parquet('data/processed/classification/primary/*.parquet')").fetchone()[0]
16+tot = con.sql("select count(*) from read_parquet('data/processed/classification/input/chunk=*/*.parquet')").fetchone()[0]
17+json.dump({"ontology_version": o.get("version"), "works": tot, "classified": n, "when": dt.datetime.now(dt.UTC).isoformat()}, open("data/processed/classification/classification_report.json", "w"), indent=1)
18+print("classified", n, "of", tot)
19+EOF2
20+echo CLASSIFY_DONE >> logs/classify/prepare.log
modified tests/test_classify_rules.py +1 −1
@@ -21,7 +21,7 @@ def test_keyword_alone_reaches_threshold_and_child_wins_tie():
21 ids = {r[0] for r in res}; assert ids == {"llm"} 21 ids = {r[0] for r in res}; assert ids == {"llm"}
22 res = classify.score_row("Foundation models and large language models", "", [10028], [], [], []) 22 res = classify.score_row("Foundation models and large language models", "", [10028], [], [], [])
23 scores = {r[0]: r[3] for r in res}; assert scores["llm"] == 1.6 and scores["foundation_models"] == 1.6 23 scores = {r[0]: r[3] for r in res}; assert scores["llm"] == 1.6 and scores["foundation_models"] == 1.6
24− parents = {r[2] for r in res if r[2]} 24+ parents = set().union(*[set(r[2]) for r in res])
25 best = sorted(res, key=lambda r: (-r[3], 1 if r[0] in parents else 0, r[1]))[0]; assert best[0] == "llm" 25 best = sorted(res, key=lambda r: (-r[3], 1 if r[0] in parents else 0, r[1]))[0]; assert best[0] == "llm"
26 26
27 27
28 28
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere