spb/uqo-research-db

1#!/bin/bash2# Full SPID LLM extraction + post-processing on the merged corpus (approved by Simon 2026-10-27).3set -u4ROOT="$HOME/UQO_RESEARCH_DB"; cd "$ROOT/project-db"5export PYTHONUNBUFFERED=16LOGS="$ROOT/data/logs"; PY="$ROOT/.venv/bin/python"7while pgrep -f "python -m spid" >/dev/null || pgrep -f "run_full_extract.py" >/dev/null; do sleep 60; done  # wait for the SPID DuckDB lock8rm -f "$LOGS/stage.spid.done" "$LOGS/stage.spid.failed"9echo "=== $(date "+%F %T") SPID full run start" >> "$LOGS/spid.log"10PYTHONPATH=. $PY -m spid pending > "$LOGS/spid_pending.json" 2>&111PYTHONPATH=. $PY run_full_extract.py --workers 48 --batch 20 >> "$LOGS/spid_extract.log" 2>&112rc=$?; echo "=== $(date "+%F %T") extract rc=$rc" >> "$LOGS/spid.log"13PYTHONPATH=. $PY -m spid pending >> "$LOGS/spid.log" 2>&114for step in resolve graph embed analytics; do15  echo "=== $(date "+%F %T") $step" >> "$LOGS/spid.log"16  PYTHONPATH=. $PY -m spid $step >> "$LOGS/spid_post.log" 2>&1 || echo "$step FAILED" >> "$LOGS/spid.log"17done18PYTHONPATH=. $PY -m spid stats >> "$LOGS/spid.log" 2>&119[ $rc -eq 0 ] && touch "$LOGS/stage.spid.done" || touch "$LOGS/stage.spid.failed"20