#!/bin/sh
# wait for the second snapshot pass, then normalise, compact, re-link, re-define the corpus and regenerate volume tables
cd ~/wp44_uqo; . .venv/bin/activate; ulimit -n 65536
while [ ! -f logs/pass2_done.flag ]; do sleep 60; done   # flag written after the pass-2 outputs (computed on M5M36) are copied back
python ingestion/build_corpus.py --cs-dir data/raw/openalex/works_pass2/cs --out data/processed_p2 --workers 12 > logs/build_corpus_p2.log 2>&1
python - <<EOF3
import os; os.makedirs("data/processed/tables_p2", exist_ok=True)
EOF3
rm -rf data/processed/tables_p2; mv data/processed_p2/tables data/processed/tables_p2
python ingestion/compact_tables.py ~/wp44_uqo _p2 > logs/compact_p2.log 2>&1
python ingestion/link_sources.py --root ~/wp44_uqo > logs/link_sources_v2.log 2>&1
python systematic_review/define_corpus.py --root ~/wp44_uqo > logs/define_corpus_v2.log 2>&1
python bibliometrics/01_volume_series.py --root ~/wp44_uqo > logs/volume_series_v2.log 2>&1
echo CHAIN_DONE >> logs/define_corpus_v2.log