SPID: registre extracted_sections, DuckDB 2.0, scripts de run/reprise (extraction Russell 3000 terminée 2026-10-28)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
3 changed files +37 −1
modified
project-db/spid/store.py
+3 −1
@@ -186,7 +186,9 @@ class Store: | |||||
| 186 | sql += " ORDER BY filing_date" | 186 | sql += " ORDER BY filing_date" | ||
| 187 | if limit: | 187 | if limit: | ||
| 188 | sql += f" LIMIT {int(limit)}" | 188 | sql += f" LIMIT {int(limit)}" | ||
| 189 | − return self.con.execute(sql).fetch_arrow_table().to_pylist() | 189 | + res = self.con.execute(sql) | ||
| 190 | + table = res.to_arrow_table() if hasattr(res, "to_arrow_table") else res.fetch_arrow_table() # DuckDB >= 2.0 / < 2.0 | ||||
| 191 | + return table.to_pylist() | ||||
| 190 | 192 | ||||
| 191 | # ---- write side --------------------------------------------------- | 193 | # ---- write side --------------------------------------------------- | ||
| 192 | def upsert_mentions(self, mentions: list[ProjectMention], ctx: dict) -> int: | 194 | def upsert_mentions(self, mentions: list[ProjectMention], ctx: dict) -> int: | ||
added
scripts/cluster/spid_resume_when_credits.sh
+15 −0
@@ -0,0 +1,15 @@ | |||||
| 1 | +#!/bin/bash | ||||
| 2 | +# Probes the OpenAI API every 10 min; once credits are back (HTTP 200) and the interim | ||||
| 3 | +# post-processing is finished, relaunches the full SPID run (pending sections only). | ||||
| 4 | +ROOT="$HOME/UQO_RESEARCH_DB"; cd "$ROOT/project-db" | ||||
| 5 | +KEY=$(grep "^OPENAI_API_KEY=" .env | cut -d= -f2) | ||||
| 6 | +while true; do | ||||
| 7 | + code=$(curl -s -o /dev/null -w "%{http_code}" https://api.openai.com/v1/chat/completions -H "Authorization: Bearer $KEY" -H "Content-Type: application/json" -d "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}],\"max_tokens\":1}") | ||||
| 8 | + echo "$(date "+%F %T") probe HTTP $code" >> "$ROOT/data/logs/spid_resume.log" | ||||
| 9 | + if [ "$code" = "200" ] && [ -f "$ROOT/data/logs/stage.spid_interim.done" ]; then | ||||
| 10 | + echo "$(date "+%F %T") credits back -> relaunching spid_run.sh" >> "$ROOT/data/logs/spid_resume.log" | ||||
| 11 | + nohup "$ROOT/scripts/cluster/spid_run.sh" > "$ROOT/data/logs/nohup.spid.out" 2>&1 < /dev/null & | ||||
| 12 | + exit 0 | ||||
| 13 | + fi | ||||
| 14 | + sleep 600 | ||||
| 15 | +done | ||||
added
scripts/cluster/spid_run.sh
+19 −0
@@ -0,0 +1,19 @@ | |||||
| 1 | +#!/bin/bash | ||||
| 2 | +# Full SPID LLM extraction + post-processing on the merged corpus (approved by Simon 2026-10-27). | ||||
| 3 | +set -u | ||||
| 4 | +ROOT="$HOME/UQO_RESEARCH_DB"; cd "$ROOT/project-db" | ||||
| 5 | +export PYTHONUNBUFFERED=1 | ||||
| 6 | +LOGS="$ROOT/data/logs"; PY="$ROOT/.venv/bin/python" | ||||
| 7 | +while pgrep -f "python -m spid" >/dev/null || pgrep -f "run_full_extract.py" >/dev/null; do sleep 60; done # wait for the SPID DuckDB lock | ||||
| 8 | +rm -f "$LOGS/stage.spid.done" "$LOGS/stage.spid.failed" | ||||
| 9 | +echo "=== $(date "+%F %T") SPID full run start" >> "$LOGS/spid.log" | ||||
| 10 | +PYTHONPATH=. $PY -m spid pending > "$LOGS/spid_pending.json" 2>&1 | ||||
| 11 | +PYTHONPATH=. $PY run_full_extract.py --workers 48 --batch 20 >> "$LOGS/spid_extract.log" 2>&1 | ||||
| 12 | +rc=$?; echo "=== $(date "+%F %T") extract rc=$rc" >> "$LOGS/spid.log" | ||||
| 13 | +PYTHONPATH=. $PY -m spid pending >> "$LOGS/spid.log" 2>&1 | ||||
| 14 | +for step in resolve graph embed analytics; do | ||||
| 15 | + echo "=== $(date "+%F %T") $step" >> "$LOGS/spid.log" | ||||
| 16 | + PYTHONPATH=. $PY -m spid $step >> "$LOGS/spid_post.log" 2>&1 || echo "$step FAILED" >> "$LOGS/spid.log" | ||||
| 17 | +done | ||||
| 18 | +PYTHONPATH=. $PY -m spid stats >> "$LOGS/spid.log" 2>&1 | ||||
| 19 | +[ $rc -eq 0 ] && touch "$LOGS/stage.spid.done" || touch "$LOGS/stage.spid.failed" | ||||
| 20 | |||||