WP44: ontology v1.0 (Agent D), classifier shard mode, taxonomy tests
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
4 changed files +35 −4
modified
agents/REGISTER.md
+1 −1
@@ -4,5 +4,5 @@ | ||
| 4 | 4 | |---|---|---|---|---|---| |
| 5 | 5 | | 2026-11-02 | A (director, this session) | CLAUDE.md | RESEARCH_PLAN.md, protocol/PROTOCOL.md, reports/PHASE0_FEASIBILITY.md | PI checkpoint H-0 pending | delivered | |
| 6 | 6 | | 2026-11-02/03 | B+C (acquisition & data engineering, this session) | RESEARCH_PLAN §2–3 | ingestion/*.py, ledgers on M3U96b, data/metadata/linkage_report.json | N (audit, Phase 8) pending | delivered | |
| 7 | −| 2026-11-03 | D (taxonomy architect, subagent, 2nd attempt) | agents/briefs/AGENT_D_TAXONOMY.md | taxonomy/ontology.yaml, topic_map.csv, ONTOLOGY.md, src/ | A + K (validation sample, Phase 2) | running | | |
| 7 | +| 2026-11-03 | D (taxonomy architect, subagent, 2nd attempt) | agents/briefs/AGENT_D_TAXONOMY.md | taxonomy/ontology.yaml (v1.0, 40 leaves, 8 areas, 646 topic weights, 771 DBLP streams, 273 regexes, 62 unmapped CS topics listed), topic_map.csv, ONTOLOGY.md, src/ | A (build re-run, classify smoke test 2026-11-03) + validation sample (Phase 2) | delivered 2026-11-03 | | |
| 8 | 8 | | 2026-11-03 | B-lit (reference acquisition, subagent) | agents/briefs/AGENT_LIT_REFERENCES.md | manuscript/references.bib, references_verification.csv, literature_synthesis/REFERENCE_MAP.md | M (compile) + N (re-verify sample) | running | |
modified
taxonomy/classify.py
+13 −2
@@ -28,7 +28,7 @@ def load_ontology(path): | ||
| 28 | 28 | kws.append((re.compile(k["pattern"]), float(k["weight"]), k.get("field", "both"))) |
| 29 | 29 | exc = [re.compile(p if isinstance(p, str) else p["pattern"]) for p in (c.get("exclusions") or [])] |
| 30 | 30 | comp.append(dict( |
| 31 | − id=c["id"], number=int(c["number"]), parent=c.get("child_of"), | |
| 31 | + id=c["id"], number=int(c["number"]), parent=tuple(c["child_of"]) if isinstance(c.get("child_of"), list) else ((c["child_of"],) if c.get("child_of") else ()), | |
| 32 | 32 | topics={str(t["id"]).lstrip("T"): float(t["weight"]) for t in (c.get("openalex_topics") or [])}, |
| 33 | 33 | subfields={str(t["id"]): float(t["weight"]) for t in (c.get("openalex_subfields") or [])}, |
| 34 | 34 | arxiv={t["id"]: float(t["weight"]) for t in (c.get("arxiv_categories") or [])}, |
@@ -81,7 +81,7 @@ def process_chunk(args): | ||
| 81 | 81 | for (cid, num, parent, s, st, ss, sa, sd, sk) in res: |
| 82 | 82 | cw.append(t["id"][i]); cc.append(cid); cs.append(s); c1.append(st); c2.append(ss); c3.append(sa); c4.append(sd); c5.append(sk) |
| 83 | 83 | if res: |
| 84 | − parents = {r[2] for r in res if r[2]} | |
| 84 | + parents = set().union(*[set(r[2]) for r in res]) | |
| 85 | 85 | # primary: highest score; tie → child before parent (a category that is someone's parent loses), then lower number |
| 86 | 86 | best = sorted(res, key=lambda r: (-r[3], 1 if r[0] in parents else 0, r[1]))[0] |
| 87 | 87 | pw.append(t["id"][i]); pc.append(best[0]); pn.append(len(res)); ps.append(best[3]) |
@@ -113,9 +113,20 @@ def main(): | ||
| 113 | 113 | ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); ap.add_argument("--ontology", required=True) |
| 114 | 114 | ap.add_argument("--prepare", action="store_true"); ap.add_argument("--run", action="store_true"); ap.add_argument("--workers", type=int, default=20) |
| 115 | 115 | ap.add_argument("--chunk-rows", type=int, default=100000); ap.add_argument("--limit", type=int, default=0) |
| 116 | + ap.add_argument("--shard", default=None, help="i/N: process only chunks with index %% N == i in this process (no pool; used by run_classify_shards.sh)") | |
| 116 | 117 | a = ap.parse_args(); out_dir = os.path.join(a.root, "data/processed/classification"); db = os.path.join(a.root, "data/processed/wp44.duckdb") |
| 117 | 118 | if a.prepare: |
| 118 | 119 | prepare(a.root, db, out_dir, a.chunk_rows) |
| 120 | + if a.run and a.shard: | |
| 121 | + i, N = map(int, a.shard.split("/")); _init(a.ontology) | |
| 122 | + files = sorted(glob.glob(os.path.join(out_dir, "input", "chunk=*", "*.parquet")), key=lambda f: int(os.path.basename(os.path.dirname(f)).split("=")[1])) | |
| 123 | + t0 = time.time(); tot = cl = 0 | |
| 124 | + for k, f in enumerate(files): | |
| 125 | + if k % N != i: | |
| 126 | + continue | |
| 127 | + name, n, c = process_chunk_named((f, out_dir, os.path.basename(os.path.dirname(f)))); tot += n; cl += c | |
| 128 | + print(f"[{time.time()-t0:6.0f}s] shard {i}/{N} {name}: {n} works, {c} classified | {tot} / {cl}", flush=True) | |
| 129 | + return | |
| 119 | 130 | if a.run: |
| 120 | 131 | files = sorted(glob.glob(os.path.join(out_dir, "input", "chunk=*", "*.parquet"))) |
| 121 | 132 | files = [(f, out_dir) for f in files] |
added
taxonomy/run_classify_shards.sh
+20 −0
@@ -0,0 +1,20 @@ | ||
| 1 | +#!/bin/sh | |
| 2 | +# Classify the corpus with N independent processes (macOS multiprocessing pool hangs with pyarrow in spawned workers). | |
| 3 | +# Usage: sh taxonomy/run_classify_shards.sh [N] — waits for the pass-2 chain (CHAIN_DONE) before preparing the input. | |
| 4 | +cd ~/wp44_uqo; . .venv/bin/activate; ulimit -n 65536; N=${1:-20} | |
| 5 | +while ! grep -q CHAIN_DONE logs/define_corpus_v2.log 2>/dev/null; do sleep 60; done | |
| 6 | +rm -rf data/processed/classification; mkdir -p logs/classify | |
| 7 | +python taxonomy/classify.py --root ~/wp44_uqo --ontology taxonomy/ontology.yaml --prepare > logs/classify/prepare.log 2>&1 | |
| 8 | +i=0; while [ $i -lt $N ]; do | |
| 9 | + nohup python taxonomy/classify.py --root ~/wp44_uqo --ontology taxonomy/ontology.yaml --run --shard $i/$N > logs/classify/shard_$i.log 2>&1 & | |
| 10 | + i=$((i+1)); done | |
| 11 | +wait | |
| 12 | +python - <<EOF2 | |
| 13 | +import duckdb, json, datetime as dt, yaml | |
| 14 | +con = duckdb.connect(); o = yaml.safe_load(open("taxonomy/ontology.yaml")) | |
| 15 | +n = con.sql("select count(distinct work_id) from read_parquet('data/processed/classification/primary/*.parquet')").fetchone()[0] | |
| 16 | +tot = con.sql("select count(*) from read_parquet('data/processed/classification/input/chunk=*/*.parquet')").fetchone()[0] | |
| 17 | +json.dump({"ontology_version": o.get("version"), "works": tot, "classified": n, "when": dt.datetime.now(dt.UTC).isoformat()}, open("data/processed/classification/classification_report.json", "w"), indent=1) | |
| 18 | +print("classified", n, "of", tot) | |
| 19 | +EOF2 | |
| 20 | +echo CLASSIFY_DONE >> logs/classify/prepare.log | |
modified
tests/test_classify_rules.py
+1 −1
@@ -21,7 +21,7 @@ def test_keyword_alone_reaches_threshold_and_child_wins_tie(): | ||
| 21 | 21 | ids = {r[0] for r in res}; assert ids == {"llm"} |
| 22 | 22 | res = classify.score_row("Foundation models and large language models", "", [10028], [], [], []) |
| 23 | 23 | scores = {r[0]: r[3] for r in res}; assert scores["llm"] == 1.6 and scores["foundation_models"] == 1.6 |
| 24 | − parents = {r[2] for r in res if r[2]} | |
| 24 | + parents = set().union(*[set(r[2]) for r in res]) | |
| 25 | 25 | best = sorted(res, key=lambda r: (-r[3], 1 if r[0] in parents else 0, r[1]))[0]; assert best[0] == "llm" |
| 26 | 26 | |
| 27 | 27 | |
| 28 | 28 | |