#!/bin/sh
# Classify the corpus with N independent local processes on M3U96b (macOS pools hang with pyarrow; remote node kept free).
# Usage: sh taxonomy/run_classify_shards.sh [N]  — waits for the pass-2 chain (CHAIN_DONE) before preparing the input.
cd ~/wp44_uqo; . .venv/bin/activate; ulimit -n 65536; N=${1:-26}
while ! grep -q CHAIN_DONE logs/define_corpus_v2.log 2>/dev/null; do sleep 60; done
rm -rf data/processed/classification; mkdir -p logs/classify
python taxonomy/classify.py --root ~/wp44_uqo --ontology taxonomy/ontology.yaml --prepare > logs/classify/prepare.log 2>&1
i=0; while [ $i -lt $N ]; do
  nohup python taxonomy/classify.py --root ~/wp44_uqo --ontology taxonomy/ontology.yaml --run --shard $i/$N > logs/classify/shard_$i.log 2>&1 &
  i=$((i+1)); done
wait
python - <<EOF2
import duckdb, json, datetime as dt, yaml
con = duckdb.connect(); o = yaml.safe_load(open("taxonomy/ontology.yaml"))
n = con.sql("select count(distinct work_id) from read_parquet('data/processed/classification/primary/*.parquet')").fetchall()[0][0]
tot = con.sql("select count(*) from read_parquet('data/processed/classification/input/chunk=*/*.parquet')").fetchall()[0][0]
json.dump({"ontology_version": o.get("version"), "works": tot, "classified": n, "when": dt.datetime.now(dt.UTC).isoformat()}, open("data/processed/classification/classification_report.json", "w"), indent=1)
print("classified", n, "of", tot)
EOF2
echo CLASSIFY_DONE >> logs/classify/prepare.log