#!/bin/bash
# Full SPID LLM extraction + post-processing on the merged corpus (approved by Simon 2026-10-27).
set -u
ROOT="$HOME/UQO_RESEARCH_DB"; cd "$ROOT/project-db"
export PYTHONUNBUFFERED=1
LOGS="$ROOT/data/logs"; PY="$ROOT/.venv/bin/python"
while pgrep -f "python -m spid" >/dev/null || pgrep -f "run_full_extract.py" >/dev/null; do sleep 60; done  # wait for the SPID DuckDB lock
rm -f "$LOGS/stage.spid.done" "$LOGS/stage.spid.failed"
echo "=== $(date "+%F %T") SPID full run start" >> "$LOGS/spid.log"
PYTHONPATH=. $PY -m spid pending > "$LOGS/spid_pending.json" 2>&1
PYTHONPATH=. $PY run_full_extract.py --workers 48 --batch 20 >> "$LOGS/spid_extract.log" 2>&1
rc=$?; echo "=== $(date "+%F %T") extract rc=$rc" >> "$LOGS/spid.log"
PYTHONPATH=. $PY -m spid pending >> "$LOGS/spid.log" 2>&1
for step in resolve graph embed analytics; do
  echo "=== $(date "+%F %T") $step" >> "$LOGS/spid.log"
  PYTHONPATH=. $PY -m spid $step >> "$LOGS/spid_post.log" 2>&1 || echo "$step FAILED" >> "$LOGS/spid.log"
done
PYTHONPATH=. $PY -m spid stats >> "$LOGS/spid.log" 2>&1
[ $rc -eq 0 ] && touch "$LOGS/stage.spid.done" || touch "$LOGS/stage.spid.failed"