#!/bin/bash
# Waits for the merged database, then runs the full SPID LLM extraction + post-processing.
set -u
ROOT="$HOME/UQO_RESEARCH_DB"; cd "$ROOT"
export PYTHONUNBUFFERED=1
LOGS="$ROOT/data/logs"
while [ ! -f "$LOGS/stage.merge.done" ]; do sleep 300; done
echo "=== $(date "+%F %T") merge done, starting SPID" > "$LOGS/spid.log"
cd project-db
PY="$ROOT/.venv/bin/python"
PYTHONPATH=. $PY -m spid pending > "$LOGS/spid_pending.json" 2>&1
cat "$LOGS/spid_pending.json" >> "$LOGS/spid.log"
PYTHONPATH=. $PY run_full_extract.py --workers 24 >> "$LOGS/spid_extract.log" 2>&1
rc=$?; echo "=== $(date "+%F %T") extract rc=$rc" >> "$LOGS/spid.log"
for step in resolve graph embed analytics; do
  echo "=== $(date "+%F %T") $step" >> "$LOGS/spid.log"
  PYTHONPATH=. $PY -m spid $step >> "$LOGS/spid_post.log" 2>&1 || echo "$step FAILED" >> "$LOGS/spid.log"
done
PYTHONPATH=. $PY -m spid stats >> "$LOGS/spid.log" 2>&1
touch "$LOGS/stage.spid.done"