spb/uqo-research-db

1#!/bin/bash2# Per-node stage runner for the Russell 3000 extension (2026-10-27).3#   run_shard.sh <node> <stage> [args]4# stages: download_self | build_db | parse | embed | export | pipeline (build_db→parse→embed→export)5# Each stage writes data/logs/<stage>.log and data/logs/stage.<stage>.{running,done,failed}.6set -u7NODE="$1"; STAGE="$2"; shift 28ROOT="$HOME/UQO_RESEARCH_DB"9cd "$ROOT" || exit 110export PATH="$HOME/.local/bin:/opt/homebrew/bin:$PATH"11export PYTHONUNBUFFERED=112PY="$ROOT/.venv/bin/python"13LOGS="$ROOT/data/logs"; mkdir -p "$LOGS" "$ROOT/data/metadata/shards"14RATE="${RATE:-0.9}"; WORKERS="${WORKERS:-8}"; DL_WORKERS="${DL_WORKERS:-6}"1516mark() { rm -f "$LOGS/stage.$1.running" "$LOGS/stage.$1.done" "$LOGS/stage.$1.failed"; touch "$LOGS/stage.$1.$2"; }1718run_stage() {  # name, command...19  local name="$1"; shift20  mark "$name" running21  echo "=== $(date '+%F %T') [$NODE] stage $name start: $*" >> "$LOGS/$name.log"22  "$@" >> "$LOGS/$name.log" 2>&123  local rc=$?24  if [ $rc -eq 0 ]; then mark "$name" done; else mark "$name" failed; fi25  echo "=== $(date '+%F %T') [$NODE] stage $name rc=$rc" >> "$LOGS/$name.log"26  return $rc27}2829case "$STAGE" in30  download_self)31    LIST="data/metadata/shards/$NODE.self.txt"32    if [ ! -s "$LIST" ]; then mark download_self done; exit 0; fi33    KNOWN=""; [ -s data/metadata/download_log.json ] && KNOWN="--known-logs data/metadata/download_log.json"34    run_stage download_self "$PY" scripts/download_filings.py --ciks-file "$LIST" --rate "$RATE" \35        --workers "$DL_WORKERS" --log data/metadata/download_log.self.jsonl $KNOWN ;;36  build_db)37    run_stage build_db "$PY" scripts/build_database.py --shard-node "$NODE" --no-parquet ;;38  parse)39    run_stage parse "$PY" scripts/parse_filings.py --max-workers "$WORKERS" --batch-size 800 --no-json ;;40  embed)41    run_stage embed "$PY" scripts/build_embeddings.py --batch-size 256 ;;42  export)43    run_stage export "$PY" scripts/export_shard.py ;;44  pipeline)45    mark pipeline running46    "$0" "$NODE" build_db && "$0" "$NODE" parse && "$0" "$NODE" embed && "$0" "$NODE" export47    rc=$?; if [ $rc -eq 0 ]; then mark pipeline done; else mark pipeline failed; fi; exit $rc ;;48  pipeline_master)49    mark pipeline running50    "$0" "$NODE" build_db && "$0" "$NODE" parse && "$0" "$NODE" embed51    rc=$?; if [ $rc -eq 0 ]; then mark pipeline done; else mark pipeline failed; fi; exit $rc ;;52  *) echo "unknown stage $STAGE"; exit 2 ;;53esac54