spb/uqo-research-db

1#!/bin/bash2# Waits for the merged database, then runs the full SPID LLM extraction + post-processing.3set -u4ROOT="$HOME/UQO_RESEARCH_DB"; cd "$ROOT"5export PYTHONUNBUFFERED=16LOGS="$ROOT/data/logs"7while [ ! -f "$LOGS/stage.merge.done" ]; do sleep 300; done8echo "=== $(date "+%F %T") merge done, starting SPID" > "$LOGS/spid.log"9cd project-db10PY="$ROOT/.venv/bin/python"11PYTHONPATH=. $PY -m spid pending > "$LOGS/spid_pending.json" 2>&112cat "$LOGS/spid_pending.json" >> "$LOGS/spid.log"13PYTHONPATH=. $PY run_full_extract.py --workers 24 >> "$LOGS/spid_extract.log" 2>&114rc=$?; echo "=== $(date "+%F %T") extract rc=$rc" >> "$LOGS/spid.log"15for step in resolve graph embed analytics; do16  echo "=== $(date "+%F %T") $step" >> "$LOGS/spid.log"17  PYTHONPATH=. $PY -m spid $step >> "$LOGS/spid_post.log" 2>&1 || echo "$step FAILED" >> "$LOGS/spid.log"18done19PYTHONPATH=. $PY -m spid stats >> "$LOGS/spid.log" 2>&120touch "$LOGS/stage.spid.done"21