1#!/bin/sh2# Classify the corpus with N independent local processes on M3U96b (macOS pools hang with pyarrow; remote node kept free).3# Usage: sh taxonomy/run_classify_shards.sh [N] — waits for the pass-2 chain (CHAIN_DONE) before preparing the input.4cd ~/wp44_uqo; . .venv/bin/activate; ulimit -n 65536; N=${1:-26}5while ! grep -q CHAIN_DONE logs/define_corpus_v2.log 2>/dev/null; do sleep 60; done6rm -rf data/processed/classification; mkdir -p logs/classify7python taxonomy/classify.py --root ~/wp44_uqo --ontology taxonomy/ontology.yaml --prepare > logs/classify/prepare.log 2>&18i=0; while [ $i -lt $N ]; do9 nohup python taxonomy/classify.py --root ~/wp44_uqo --ontology taxonomy/ontology.yaml --run --shard $i/$N > logs/classify/shard_$i.log 2>&1 &10 i=$((i+1)); done11wait12python - <<EOF213import duckdb, json, datetime as dt, yaml14con = duckdb.connect(); o = yaml.safe_load(open("taxonomy/ontology.yaml"))15n = con.sql("select count(distinct work_id) from read_parquet('data/processed/classification/primary/*.parquet')").fetchone()[0]16tot = con.sql("select count(*) from read_parquet('data/processed/classification/input/chunk=*/*.parquet')").fetchone()[0]17json.dump({"ontology_version": o.get("version"), "works": tot, "classified": n, "when": dt.datetime.now(dt.UTC).isoformat()}, open("data/processed/classification/classification_report.json", "w"), indent=1)18print("classified", n, "of", tot)19EOF220echo CLASSIFY_DONE >> logs/classify/prepare.log21