SPID: registre extracted_sections, DuckDB 2.0, scripts de run/reprise (extraction Russell 3000 terminée 2026-10-28)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
3 changed files +37 −1
modified
project-db/spid/store.py
+3 −1
@@ -186,7 +186,9 @@ class Store: | ||
| 186 | 186 | sql += " ORDER BY filing_date" |
| 187 | 187 | if limit: |
| 188 | 188 | sql += f" LIMIT {int(limit)}" |
| 189 | − return self.con.execute(sql).fetch_arrow_table().to_pylist() | |
| 189 | + res = self.con.execute(sql) | |
| 190 | + table = res.to_arrow_table() if hasattr(res, "to_arrow_table") else res.fetch_arrow_table() # DuckDB >= 2.0 / < 2.0 | |
| 191 | + return table.to_pylist() | |
| 190 | 192 | |
| 191 | 193 | # ---- write side --------------------------------------------------- |
| 192 | 194 | def upsert_mentions(self, mentions: list[ProjectMention], ctx: dict) -> int: |
added
scripts/cluster/spid_resume_when_credits.sh
+15 −0
@@ -0,0 +1,15 @@ | ||
| 1 | +#!/bin/bash | |
| 2 | +# Probes the OpenAI API every 10 min; once credits are back (HTTP 200) and the interim | |
| 3 | +# post-processing is finished, relaunches the full SPID run (pending sections only). | |
| 4 | +ROOT="$HOME/UQO_RESEARCH_DB"; cd "$ROOT/project-db" | |
| 5 | +KEY=$(grep "^OPENAI_API_KEY=" .env | cut -d= -f2) | |
| 6 | +while true; do | |
| 7 | + code=$(curl -s -o /dev/null -w "%{http_code}" https://api.openai.com/v1/chat/completions -H "Authorization: Bearer $KEY" -H "Content-Type: application/json" -d "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}],\"max_tokens\":1}") | |
| 8 | + echo "$(date "+%F %T") probe HTTP $code" >> "$ROOT/data/logs/spid_resume.log" | |
| 9 | + if [ "$code" = "200" ] && [ -f "$ROOT/data/logs/stage.spid_interim.done" ]; then | |
| 10 | + echo "$(date "+%F %T") credits back -> relaunching spid_run.sh" >> "$ROOT/data/logs/spid_resume.log" | |
| 11 | + nohup "$ROOT/scripts/cluster/spid_run.sh" > "$ROOT/data/logs/nohup.spid.out" 2>&1 < /dev/null & | |
| 12 | + exit 0 | |
| 13 | + fi | |
| 14 | + sleep 600 | |
| 15 | +done | |
added
scripts/cluster/spid_run.sh
+19 −0
@@ -0,0 +1,19 @@ | ||
| 1 | +#!/bin/bash | |
| 2 | +# Full SPID LLM extraction + post-processing on the merged corpus (approved by Simon 2026-10-27). | |
| 3 | +set -u | |
| 4 | +ROOT="$HOME/UQO_RESEARCH_DB"; cd "$ROOT/project-db" | |
| 5 | +export PYTHONUNBUFFERED=1 | |
| 6 | +LOGS="$ROOT/data/logs"; PY="$ROOT/.venv/bin/python" | |
| 7 | +while pgrep -f "python -m spid" >/dev/null || pgrep -f "run_full_extract.py" >/dev/null; do sleep 60; done # wait for the SPID DuckDB lock | |
| 8 | +rm -f "$LOGS/stage.spid.done" "$LOGS/stage.spid.failed" | |
| 9 | +echo "=== $(date "+%F %T") SPID full run start" >> "$LOGS/spid.log" | |
| 10 | +PYTHONPATH=. $PY -m spid pending > "$LOGS/spid_pending.json" 2>&1 | |
| 11 | +PYTHONPATH=. $PY run_full_extract.py --workers 48 --batch 20 >> "$LOGS/spid_extract.log" 2>&1 | |
| 12 | +rc=$?; echo "=== $(date "+%F %T") extract rc=$rc" >> "$LOGS/spid.log" | |
| 13 | +PYTHONPATH=. $PY -m spid pending >> "$LOGS/spid.log" 2>&1 | |
| 14 | +for step in resolve graph embed analytics; do | |
| 15 | + echo "=== $(date "+%F %T") $step" >> "$LOGS/spid.log" | |
| 16 | + PYTHONPATH=. $PY -m spid $step >> "$LOGS/spid_post.log" 2>&1 || echo "$step FAILED" >> "$LOGS/spid.log" | |
| 17 | +done | |
| 18 | +PYTHONPATH=. $PY -m spid stats >> "$LOGS/spid.log" 2>&1 | |
| 19 | +[ $rc -eq 0 ] && touch "$LOGS/stage.spid.done" || touch "$LOGS/stage.spid.failed" | |
| 20 | ||