SPB Git
spboucher.ai

spb/firc0001_uqo

Public

N0: audit infrastructure (read-only helpers, hash-verified 2324b58 snapshot, spid import shim, schema probe)

Stage D node batch N0 for FIRC-0001. sha256 of the nine snapshot files equals the node copies
(extract/llm.py, rules.py, base.py, taxonomy.py, resolve.py mtime 2026-06-17); both DuckDB files
open read_only=True; TEMP tables allowed; DuckDB temp_directory redirected to ~/firc0001_review/tmp.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Nov 5, 2026) parent 678d233 Browse files

23 changed files +3,242 −0

23 changed files
Load all 23 files Unified Split
added code/audit/a00_probe_schema.py +27 −0
@@ -0,0 +1,27 @@
1 +"""N0 probe: open both DuckDB files read-only, describe the tables used by N1-N13."""
2 +import sys, json
3 +from pathlib import Path
4 +sys.path.insert(0, str(Path(__file__).resolve().parent))
5 +from common import *
6 +
7 +r = Run("n0_probe")
8 +con = connect(SPID_DB)
9 +out = {}
10 +for t in ["extracted_sections", "project_mentions", "projects", "project_timeline", "spid_sections", "v_sections", "src_filings", "src_sections", "src_companies"]:
11 + out[t] = [tuple(x[:2]) for x in con.execute(f"DESCRIBE {t}").fetchall()]
12 +out["views"] = con.execute("select view_name, sql from duckdb_views() where not internal").fetchall()
13 +out["counts"] = {t: con.execute(f"select count(*) from {t}").fetchone()[0] for t in ["extracted_sections", "project_mentions", "projects", "project_timeline", "spid_sections"]}
14 +out["ledger_words"] = con.execute("select count(*), sum(v.word_count) from extracted_sections e join (select section_id, word_count from src_sections union all select section_id, word_count from spid_sections) v using(section_id)").fetchone()
15 +out["parquet_files"] = con.execute("select count(*), sum(length(text))/1e9 from src_sections").fetchone()
16 +e = connect(EDGAR_DB)
17 +for t in ["filings", "filing_sections", "companies", "annual_panel"]:
18 + out["edgar_"+t] = [tuple(x[:2]) for x in e.execute(f"DESCRIBE {t}").fetchall()]
19 +out["edgar_tables"] = [x[0] for x in e.execute("select table_name from duckdb_tables() where not internal").fetchall()]
20 +r.write_json("schema.json", out)
21 +# can we create a temp table on a read_only connection?
22 +try:
23 + con.execute("CREATE TEMP TABLE _t AS SELECT 1 AS x"); out["temp_ok"] = True
24 +except Exception as ex:
25 + out["temp_ok"] = str(ex)
26 +r.write_json("schema.json", out)
27 +r.finish()
added code/audit/common.py +116 −0
@@ -0,0 +1,116 @@
1 +"""Shared helpers for the FIRC-0001 Stage D read-only audits (run on M3U96b).
2 +
3 +Every script opens the two DuckDB files with read_only=True, SET threads=4,
4 +SET memory_limit='24GB', writes CSV/JSON under ~/firc0001_review/results/<name>/
5 +and a manifest.json (script sha256, DuckDB mtimes, start/end, row counts).
6 +Nothing is ever written under ~/UQO_RESEARCH_DB.
7 +"""
8 +from __future__ import annotations
9 +
10 +import csv
11 +import datetime as dt
12 +import hashlib
13 +import json
14 +import os
15 +import sys
16 +import time
17 +from pathlib import Path
18 +
19 +import duckdb
20 +
21 +HOME = Path(os.environ.get("HOME", "~")).expanduser()
22 +RDB = HOME / "UQO_RESEARCH_DB"
23 +SPID_DB = RDB / "project-db" / "data" / "db" / "spid.duckdb"
24 +EDGAR_DB = RDB / "data" / "duckdb" / "edgar_research.duckdb"
25 +WORK = HOME / "firc0001_review"
26 +RESULTS = WORK / "results"
27 +THREADS = int(os.environ.get("FIRC_THREADS", "4"))
28 +MEMLIMIT = os.environ.get("FIRC_MEMLIMIT", "24GB")
29 +SEED = 20261105
30 +
31 +
32 +def connect(path: Path = SPID_DB) -> duckdb.DuckDBPyConnection:
33 + con = duckdb.connect(str(path), read_only=True)
34 + con.execute(f"SET threads={THREADS}")
35 + con.execute(f"SET memory_limit='{MEMLIMIT}'")
36 + con.execute("SET preserve_insertion_order=false")
37 + # spill files must never land next to the databases (default = <db>.tmp)
38 + (WORK / "tmp").mkdir(parents=True, exist_ok=True)
39 + con.execute(f"SET temp_directory='{WORK / 'tmp'}'")
40 + return con
41 +
42 +
43 +def attach_edgar(con: duckdb.DuckDBPyConnection, alias: str = "edgar") -> None:
44 + con.execute(f"ATTACH '{EDGAR_DB}' AS {alias} (READ_ONLY)")
45 +
46 +
47 +def sha256_file(p: Path) -> str:
48 + h = hashlib.sha256()
49 + with open(p, "rb") as f:
50 + for chunk in iter(lambda: f.read(1 << 20), b""):
51 + h.update(chunk)
52 + return h.hexdigest()
53 +
54 +
55 +class Run:
56 + """Context for one audit script: output dir, timing, manifest."""
57 +
58 + def __init__(self, name: str, script: str | None = None, out: str | None = None):
59 + self.name = name
60 + self.script = Path(script or sys.argv[0]).resolve()
61 + self.out = Path(out) if out else RESULTS / name
62 + self.out.mkdir(parents=True, exist_ok=True)
63 + self.t0 = time.time()
64 + self.started = dt.datetime.now().isoformat(timespec="seconds")
65 + self.rows: dict[str, int] = {}
66 + self.notes: dict = {}
67 +
68 + def write_csv(self, fname: str, cols: list[str], rows) -> int:
69 + rows = list(rows)
70 + with open(self.out / fname, "w", newline="") as f:
71 + w = csv.writer(f)
72 + w.writerow(cols)
73 + w.writerows(rows)
74 + self.rows[fname] = len(rows)
75 + print(f"[{self.name}] {fname}: {len(rows)} rows", flush=True)
76 + return len(rows)
77 +
78 + def dump(self, fname: str, con, sql: str, params=None):
79 + cur = con.execute(sql, params) if params else con.execute(sql)
80 + cols = [d[0] for d in cur.description]
81 + return self.write_csv(fname, cols, cur.fetchall())
82 +
83 + def write_json(self, fname: str, obj) -> None:
84 + with open(self.out / fname, "w") as f:
85 + json.dump(obj, f, indent=2, default=str)
86 + print(f"[{self.name}] {fname} written", flush=True)
87 +
88 + def finish(self, extra: dict | None = None) -> None:
89 + man = {
90 + "package": self.name,
91 + "script": str(self.script),
92 + "script_sha256": sha256_file(self.script) if self.script.exists() else None,
93 + "started": self.started,
94 + "ended": dt.datetime.now().isoformat(timespec="seconds"),
95 + "wall_seconds": round(time.time() - self.t0, 1),
96 + "node": os.uname().nodename,
97 + "python": sys.version.split()[0],
98 + "duckdb": duckdb.__version__,
99 + "threads": THREADS,
100 + "memory_limit": MEMLIMIT,
101 + "seed": SEED,
102 + "databases": {
103 + str(SPID_DB): {"bytes": SPID_DB.stat().st_size,
104 + "mtime": dt.datetime.fromtimestamp(SPID_DB.stat().st_mtime).isoformat()},
105 + str(EDGAR_DB): {"bytes": EDGAR_DB.stat().st_size,
106 + "mtime": dt.datetime.fromtimestamp(EDGAR_DB.stat().st_mtime).isoformat()},
107 + },
108 + "read_only": True,
109 + "row_counts": self.rows,
110 + "notes": self.notes,
111 + }
112 + if extra:
113 + man.update(extra)
114 + with open(self.out / "manifest.json", "w") as f:
115 + json.dump(man, f, indent=2, default=str)
116 + print(f"[{self.name}] done in {man['wall_seconds']} s", flush=True)
added code/audit/run_all.sh +12 −0
@@ -0,0 +1,12 @@
1 +#!/bin/bash
2 +# Stage D node batch for FIRC-0001: runs N1..N13 in the plan's order, sequentially, read-only.
3 +# usage: bash run_all.sh (on M3U96b, from ~/firc0001_review; scripts/ holds the aNN_*.py copies)
4 +set -u
5 +cd "$HOME/firc0001_review"
6 +ORDER="a01_ledger_reconciliation.py a03_resolver_speccurve.py a04_runs_segmenters.py a04_git_evidence.sh a02_grounding.py a05_focus_coverage.py a06_false_positive_classes.py a09_timeseries_cohort.py a10_universe_artefacts.py a07_8k_item_benchmark.py a08_rules_baseline.py a11_annotation_frame.py a12_env_lock.sh a13_export_stats_check.sh"
7 +for S in $ORDER; do
8 + bash scripts/run_one.sh "$S"
9 + NAME=$(basename "$S" | sed -E 's/\.(py|sh)$//')
10 + while [ ! -f "logs/$NAME.done" ]; do sleep 60; done
11 + cat "logs/$NAME.done"
12 +done
added code/audit/run_one.sh +13 −0
@@ -0,0 +1,13 @@
1 +#!/bin/bash
2 +# Run one audit script on M3U96b in the background under nice, with a log and a done-marker.
3 +# usage: run_one.sh <script.py|script.sh> [args...] (from ~/firc0001_review)
4 +set -u
5 +cd "$HOME/firc0001_review"
6 +S="$1"; shift
7 +NAME=$(basename "$S" | sed -E 's/\.(py|sh)$//')
8 +LOG="logs/$NAME.log"; DONE="logs/$NAME.done"
9 +rm -f "$DONE"
10 +PY="$HOME/UQO_RESEARCH_DB/.venv/bin/python"
11 +if [[ "$S" == *.sh ]]; then CMD="bash scripts/$S $*"; else CMD="$PY -I scripts/$S $*"; fi
12 +nohup bash -c "echo START \$(date -Iseconds); nice -n 10 env OMP_NUM_THREADS=4 FIRC_THREADS=4 $CMD; echo RC=\$? END \$(date -Iseconds) > $DONE" > "$LOG" 2>&1 &
13 +echo "started $NAME pid=$! log=$LOG"
added code/audit/spid/__init__.py +1 −0
@@ -0,0 +1 @@
1 +"""spid package shim: hash-verified snapshot copies of the 2324b58 extractor modules (see spid_snapshot/SHA256SUMS)."""
added code/audit/spid/config.py +126 −0
@@ -0,0 +1,126 @@
1 +# ============================================================
2 +# SPID — configuration & paths
3 +# ============================================================
4 +from __future__ import annotations
5 +
6 +import logging
7 +import os
8 +from dataclasses import dataclass, field
9 +from functools import lru_cache
10 +from pathlib import Path
11 +
12 +# project-db/ (this file lives in project-db/spid/config.py)
13 +PROJECT_ROOT = Path(__file__).resolve().parent.parent
14 +
15 +
16 +def _load_dotenv() -> None:
17 + """Minimal .env loader (no python-dotenv dependency required)."""
18 + env_path = PROJECT_ROOT / ".env"
19 + if not env_path.exists():
20 + return
21 + for line in env_path.read_text().splitlines():
22 + line = line.strip()
23 + if not line or line.startswith("#") or "=" not in line:
24 + continue
25 + key, _, val = line.partition("=")
26 + key, val = key.strip(), val.strip().strip('"').strip("'")
27 + os.environ.setdefault(key, val)
28 +
29 +
30 +_load_dotenv()
31 +# The parent UQO_RESEARCH_DB that holds the parsed corpus.
32 +RESEARCH_ROOT = PROJECT_ROOT.parent
33 +SOURCE_DATA = RESEARCH_ROOT / "data"
34 +
35 +
36 +def _parse_scope(spec: str) -> dict:
37 + out: dict = {}
38 + for part in (spec or "").split(";"):
39 + part = part.strip()
40 + if not part or ":" not in part:
41 + continue
42 + form, items = part.split(":", 1)
43 + out[form.strip()] = [i.strip() for i in items.split(",") if i.strip()]
44 + return out
45 +
46 +
47 +@dataclass(frozen=True)
48 +class Settings:
49 + """Resolved runtime configuration for the SPID platform."""
50 +
51 + # --- source corpus (read-only) -------------------------------------
52 + source_data: Path = SOURCE_DATA
53 + parquet_dir: Path = SOURCE_DATA / "parquet"
54 + processed_dir: Path = SOURCE_DATA / "processed"
55 +
56 + @property
57 + def filings_parquet(self) -> Path:
58 + return self.parquet_dir / "filings" / "filings.parquet"
59 +
60 + @property
61 + def sections_parquet(self) -> Path:
62 + return self.parquet_dir / "filing_sections" / "filing_sections.parquet"
63 +
64 + @property
65 + def companies_parquet(self) -> Path:
66 + return self.parquet_dir / "companies" / "companies.parquet"
67 +
68 + # --- SPID outputs (writable) ---------------------------------------
69 + db_path: Path = PROJECT_ROOT / "data" / "db" / "spid.duckdb"
70 + out_dir: Path = PROJECT_ROOT / "data"
71 +
72 + # --- extraction ----------------------------------------------------
73 + backend: str = field(
74 + default_factory=lambda: os.environ.get("SPID_BACKEND", "llm")
75 + ) # "llm" (gpt-4o-mini) | "rules"
76 + min_confidence: float = 0.45
77 + # which parsed sections feed the extractor, per form (None/empty = every section)
78 + section_scope: dict = field(default_factory=lambda: _parse_scope(os.environ.get(
79 + "SPID_SECTION_SCOPE",
80 + "10-K:business,risk_factors,properties,md_and_a,quantitative_disclosures")))
81 + max_section_chars: int = 600_000
82 +
83 + # --- embeddings ----------------------------------------------------
84 + embedding_model: str = field(
85 + default_factory=lambda: os.environ.get("SPID_EMBED_MODEL", "all-MiniLM-L6-v2")
86 + )
87 +
88 + # --- LLM backend (OpenAI-compatible) -------------------------------
89 + llm_base_url: str = field(
90 + default_factory=lambda: os.environ.get(
91 + "SPID_LLM_BASE_URL", "https://api.openai.com/v1"
92 + )
93 + )
94 + llm_model: str = field(
95 + default_factory=lambda: os.environ.get("SPID_LLM_MODEL", "gpt-4o-mini")
96 + )
97 + llm_api_key: str = field(
98 + default_factory=lambda: os.environ.get("OPENAI_API_KEY", "")
99 + or os.environ.get("MACLUSTR_API_KEY", "")
100 + )
101 + llm_timeout: float = 120.0
102 +
103 + def ensure_dirs(self) -> None:
104 + self.db_path.parent.mkdir(parents=True, exist_ok=True)
105 + (self.out_dir / "exports").mkdir(parents=True, exist_ok=True)
106 +
107 +
108 +@lru_cache(maxsize=1)
109 +def get_settings() -> Settings:
110 + s = Settings()
111 + s.ensure_dirs()
112 + return s
113 +
114 +
115 +def get_logger(name: str) -> logging.Logger:
116 + logger = logging.getLogger(f"spid.{name}")
117 + if not logger.handlers:
118 + h = logging.StreamHandler()
119 + h.setFormatter(
120 + logging.Formatter("%(asctime)s %(levelname)-7s %(name)s | %(message)s",
121 + datefmt="%H:%M:%S")
122 + )
123 + logger.addHandler(h)
124 + logger.setLevel(os.environ.get("SPID_LOG_LEVEL", "INFO"))
125 + logger.propagate = False
126 + return logger
added code/audit/spid/extract/__init__.py +0 −0
added code/audit/spid/extract/base.py +69 −0
@@ -0,0 +1,69 @@
1 +# ============================================================
2 +# SPID — extraction interface & record schema
3 +# ============================================================
4 +from __future__ import annotations
5 +
6 +import hashlib
7 +from dataclasses import dataclass, field, asdict
8 +from typing import Protocol
9 +
10 +
11 +@dataclass
12 +class ProjectMention:
13 + """A single project signal extracted from one filing section.
14 +
15 + Mentions are the atomic unit; `resolve.py` merges many mentions
16 + (across filings & years) into a single Project with a timeline.
17 + """
18 + mention_id: str
19 + accession_number: str
20 + cik: str
21 + section_id: str
22 + section_name: str
23 +
24 + project_name: str
25 + project_type: str
26 + description: str
27 + objective: str = ""
28 +
29 + amount_usd: float | None = None
30 + amount_raw: str | None = None
31 + locations: list[str] = field(default_factory=list)
32 + technologies: list[str] = field(default_factory=list)
33 + partners: list[str] = field(default_factory=list)
34 + suppliers: list[str] = field(default_factory=list)
35 + benefits: list[str] = field(default_factory=list)
36 + risks: list[str] = field(default_factory=list)
37 + years: list[int] = field(default_factory=list)
38 + status: str = "mentioned"
39 +
40 + confidence: float = 0.0
41 + backend: str = "rules"
42 +
43 + def to_row(self) -> dict:
44 + d = asdict(self)
45 + # store list fields as JSON-friendly via duckdb list[str]
46 + return d
47 +
48 + @staticmethod
49 + def make_id(accession: str, section_id: str, ptype: str, anchor: str) -> str:
50 + h = hashlib.sha1(f"{accession}|{section_id}|{ptype}|{anchor}".encode()).hexdigest()
51 + return h[:20]
52 +
53 +
54 +class Extractor(Protocol):
55 + name: str
56 +
57 + def extract(self, *, text: str, accession_number: str, cik: str,
58 + section_id: str, section_name: str) -> list[ProjectMention]:
59 + ...
60 +
61 +
62 +def get_extractor(backend: str):
63 + """Factory. `backend` in {'rules','llm'}."""
64 + backend = (backend or "rules").lower()
65 + if backend == "llm":
66 + from spid.extract.llm import LLMExtractor
67 + return LLMExtractor()
68 + from spid.extract.rules import RuleExtractor
69 + return RuleExtractor()
added code/audit/spid/extract/llm.py +202 −0
@@ -0,0 +1,202 @@
1 +# ============================================================
2 +# SPID — LLM extractor (OpenAI-compatible; default gpt-4o-mini)
3 +#
4 +# Uses the chat-completions JSON mode to extract structured
5 +# project records. Cost-controlled: only project-relevant
6 +# windows of each section are sent to the model.
7 +# ============================================================
8 +from __future__ import annotations
9 +
10 +import json
11 +import re
12 +import time
13 +
14 +import httpx
15 +
16 +from spid import taxonomy as tx
17 +from spid.config import get_logger, get_settings
18 +from spid.extract.base import ProjectMention
19 +from spid.extract.rules import _PROJECT_GATE
20 +
21 +log = get_logger("extract.llm")
22 +
23 +_SYSTEM = """You are a financial analyst extracting CORPORATE PROJECTS from SEC filing text.
24 +A "project" is any concrete initiative, investment, program, expansion, construction,
25 +product/R&D effort, acquisition/integration, technology deployment, partnership, or
26 +strategic program the company is pursuing.
27 +
28 +Return STRICT JSON: {"projects": [ ... ]}. Each project object has:
29 + project_name : short descriptive name (use the official name if one is stated)
30 + project_type : EXACTLY ONE of the allowed types below
31 + description : 1 sentence, grounded in the text (no speculation)
32 + objective : the goal/why (or "")
33 + status : one of planned | in_progress | completed | mentioned
34 + amount_usd : numeric USD figure if stated (e.g. 1500000000), else null
35 + locations : array of places (city/state/country), else []
36 + technologies : array of technologies involved, else []
37 + partners : array of named partners/JV counterparties, else []
38 + benefits : array of expected benefits, else []
39 + risks : array of stated risks, else []
40 + years : array of 4-digit years mentioned, else []
41 + confidence : 0.0-1.0 (how clearly this is a real, concrete project)
42 +
43 +Allowed project_type values:
44 +%s
45 +
46 +Rules:
47 +- Extract ONLY real initiatives, not generic risk-factor boilerplate or hypotheticals.
48 +- If the text describes no concrete project, return {"projects": []}.
49 +- Do not invent facts. Numbers/locations must appear in the text.
50 +- Prefer fewer high-quality projects over many vague ones.""" % "\n".join(
51 + f" - {k}: {v}" for k, v in tx.TYPE_LABELS.items()
52 +)
53 +
54 +
55 +def _focus(text: str, budget: int = 9000) -> str:
56 + """Keep only windows around project-signal hits to bound token cost."""
57 + if len(text) <= budget:
58 + return text
59 + spans: list[tuple[int, int]] = []
60 + for m in _PROJECT_GATE.finditer(text):
61 + spans.append((max(0, m.start() - 220), min(len(text), m.end() + 380)))
62 + if sum(b - a for a, b in spans) > budget * 1.4:
63 + break
64 + if not spans:
65 + return text[:budget]
66 + # merge overlapping windows
67 + spans.sort()
68 + merged = [spans[0]]
69 + for a, b in spans[1:]:
70 + if a <= merged[-1][1]:
71 + merged[-1] = (merged[-1][0], max(merged[-1][1], b))
72 + else:
73 + merged.append((a, b))
74 + chunks, total = [], 0
75 + for a, b in merged:
76 + chunks.append(text[a:b])
77 + total += b - a
78 + if total >= budget:
79 + break
80 + return "\n...\n".join(chunks)
81 +
82 +
83 +class LLMExtractor:
84 + name = "llm"
85 +
86 + def __init__(self) -> None:
87 + self.s = get_settings()
88 + if not self.s.llm_api_key:
89 + raise RuntimeError("No LLM API key configured (OPENAI_API_KEY / MACLUSTR_API_KEY).")
90 + self._client = httpx.Client(
91 + base_url=self.s.llm_base_url,
92 + headers={"Authorization": f"Bearer {self.s.llm_api_key}"},
93 + timeout=self.s.llm_timeout,
94 + )
95 +
96 + def _call(self, content: str, retries: int = 3) -> dict:
97 + payload = {
98 + "model": self.s.llm_model,
99 + "messages": [
100 + {"role": "system", "content": _SYSTEM},
101 + {"role": "user", "content": content},
102 + ],
103 + "temperature": 0,
104 + "response_format": {"type": "json_object"},
105 + "max_tokens": 2000,
106 + }
107 + last = None
108 + for attempt in range(retries):
109 + try:
110 + r = self._client.post("/chat/completions", json=payload)
111 + if r.status_code == 429 or r.status_code >= 500:
112 + raise httpx.HTTPStatusError("retryable", request=r.request, response=r)
113 + r.raise_for_status()
114 + txt = r.json()["choices"][0]["message"]["content"]
115 + return json.loads(txt)
116 + except (httpx.HTTPError, json.JSONDecodeError, KeyError) as e:
117 + last = e
118 + time.sleep(1.5 * (attempt + 1))
119 + log.warning("LLM call failed after %d retries: %s", retries, last)
120 + return {"projects": []}
121 +
122 + def extract(self, *, text: str, accession_number: str, cik: str,
123 + section_id: str, section_name: str) -> list[ProjectMention]:
124 + if not text or not _PROJECT_GATE.search(text):
125 + return []
126 + content = (f"SECTION: {section_name}\n\n" + _focus(text))[:14000]
127 + data = self._call(content)
128 + out: list[ProjectMention] = []
129 + for p in data.get("projects", []) or []:
130 + if not isinstance(p, dict):
131 + continue
132 + ptype = self._norm_type(p.get("project_type", ""))
133 + desc = str(p.get("description", "")).strip()[:600]
134 + if not desc:
135 + continue
136 + amount = p.get("amount_usd")
137 + try:
138 + amount = float(amount) if amount not in (None, "", "null") else None
139 + except (TypeError, ValueError):
140 + amount = None
141 + anchor = f"{ptype}:{str(p.get('project_name',''))[:40]}"
142 + conf = p.get("confidence", 0.6)
143 + try:
144 + conf = float(conf)
145 + except (TypeError, ValueError):
146 + conf = 0.6
147 + out.append(ProjectMention(
148 + mention_id=ProjectMention.make_id(accession_number, section_id, ptype, anchor),
149 + accession_number=accession_number, cik=cik,
150 + section_id=section_id, section_name=section_name,
151 + project_name=str(p.get("project_name") or tx.TYPE_LABELS.get(ptype, ptype))[:200],
152 + project_type=ptype, description=desc,
153 + objective=str(p.get("objective", ""))[:400],
154 + amount_usd=amount,
155 + amount_raw=(f"${amount:,.0f}" if amount else None),
156 + locations=_strlist(p.get("locations")),
157 + technologies=_strlist(p.get("technologies")) or tx.extract_technologies(desc),
158 + partners=_strlist(p.get("partners")),
159 + benefits=_strlist(p.get("benefits")),
160 + risks=_strlist(p.get("risks")),
161 + years=_intlist(p.get("years")) or tx.extract_years(desc),
162 + status=self._norm_status(p.get("status")),
163 + confidence=round(max(0.0, min(1.0, conf)), 3),
164 + backend=self.name,
165 + ))
166 + return out
167 +
168 + @staticmethod
169 + def _norm_type(raw: str) -> str:
170 + raw = (raw or "").strip().lower().replace(" ", "_").replace("-", "_")
171 + if raw in tx.PROJECT_TYPES:
172 + return raw
173 + # fuzzy contains match against labels/keys
174 + for k, label in tx.TYPE_LABELS.items():
175 + if raw and (raw in k or raw in label.lower().replace(" ", "_")):
176 + return k
177 + return "technology_deployment"
178 +
179 + @staticmethod
180 + def _norm_status(raw) -> str:
181 + raw = (str(raw or "")).strip().lower()
182 + return raw if raw in ("planned", "in_progress", "completed", "mentioned") else "mentioned"
183 +
184 +
185 +def _strlist(v) -> list[str]:
186 + if not v:
187 + return []
188 + if isinstance(v, str):
189 + v = [v]
190 + return [str(x).strip()[:120] for x in v if str(x).strip()][:8]
191 +
192 +
193 +def _intlist(v) -> list[int]:
194 + out = []
195 + for x in (v or []):
196 + try:
197 + y = int(str(x)[:4])
198 + if 1980 <= y <= 2040:
199 + out.append(y)
200 + except (TypeError, ValueError):
201 + continue
202 + return sorted(set(out))
added code/audit/spid/extract/rules.py +120 −0
@@ -0,0 +1,120 @@
1 +# ============================================================
2 +# SPID — deterministic rule-based extractor
3 +#
4 +# High-precision baseline that needs NO LLM. Scans section text
5 +# sentence-by-sentence for project signals, classifies the type,
6 +# and pulls money / location / tech / year / status.
7 +# ============================================================
8 +from __future__ import annotations
9 +
10 +import re
11 +
12 +from spid import taxonomy as tx
13 +from spid.extract.base import ProjectMention
14 +
15 +# split into sentence-ish units; keep it cheap for 100k+ sections
16 +_SENT_SPLIT = re.compile(r"(?<=[.;:!?])\s+(?=[A-Z(\"$])")
17 +_WS = re.compile(r"\s+")
18 +# generic "project-ish" gate so we skip boilerplate sentences fast
19 +_PROJECT_GATE = re.compile(
20 + r"\b(project|initiative|program|invest|expan|construct|build|launch|develop|"
21 + r"deploy|implement|acqui|partner|facilit|capacity|capital|migrat|transform|"
22 + r"pipeline|plant|data ?cent|automat|sustainab|renewable|restructur)\w*",
23 + re.IGNORECASE,
24 +)
25 +_NAMED_PROJECT = re.compile(
26 + r"\b((?:Project|Program|Initiative)\s+[A-Z][A-Za-z0-9\-]+"
27 + r"|[A-Z][A-Za-z0-9\-]+\s+(?:Project|Program|Initiative))\b"
28 +)
29 +
30 +
31 +def _clean(s: str) -> str:
32 + return _WS.sub(" ", s).strip()
33 +
34 +
35 +class RuleExtractor:
36 + name = "rules"
37 +
38 + def extract(self, *, text: str, accession_number: str, cik: str,
39 + section_id: str, section_name: str) -> list[ProjectMention]:
40 + if not text:
41 + return []
42 + text = text[:600_000]
43 + out: list[ProjectMention] = []
44 + seen: set[str] = set()
45 +
46 + for raw_sent in _SENT_SPLIT.split(text):
47 + sent = _clean(raw_sent)
48 + if not (20 <= len(sent) <= 700):
49 + continue
50 + if not _PROJECT_GATE.search(sent):
51 + continue
52 +
53 + low = sent.lower()
54 + matched_types: list[str] = []
55 + for ptype, regexes in tx.TYPE_SIGNALS.items():
56 + if any(rx.search(low) for rx in regexes):
57 + matched_types.append(ptype)
58 + if not matched_types:
59 + continue
60 +
61 + amount, amount_raw = tx.extract_money(sent)
62 + locations = tx.extract_locations(sent)
63 + techs = tx.extract_technologies(sent)
64 + years = tx.extract_years(sent)
65 + status = tx.detect_status(sent)
66 + nm = _NAMED_PROJECT.search(sent)
67 +
68 + # keep at most the 2 strongest types for one sentence
69 + for ptype in matched_types[:2]:
70 + anchor = f"{ptype}:{(locations[0] if locations else '')}:{int(amount or 0)}"
71 + if anchor in seen:
72 + continue
73 + seen.add(anchor)
74 +
75 + conf = self._confidence(ptype, amount, locations, years, status, nm, low)
76 + if conf < 0.30:
77 + continue
78 +
79 + pname = nm.group(1) if nm else self._synth_name(ptype, locations, techs)
80 + out.append(ProjectMention(
81 + mention_id=ProjectMention.make_id(accession_number, section_id, ptype, anchor),
82 + accession_number=accession_number, cik=cik,
83 + section_id=section_id, section_name=section_name,
84 + project_name=pname, project_type=ptype,
85 + description=sent[:500],
86 + amount_usd=amount, amount_raw=amount_raw,
87 + locations=locations, technologies=techs, years=years,
88 + status=status, confidence=round(conf, 3), backend=self.name,
89 + ))
90 + return out
91 +
92 + @staticmethod
93 + def _synth_name(ptype: str, locations: list[str], techs: list[str]) -> str:
94 + label = tx.TYPE_LABELS.get(ptype, ptype)
95 + if locations:
96 + return f"{label} — {locations[0]}"
97 + if techs:
98 + return f"{label} ({techs[0]})"
99 + return label
100 +
101 + @staticmethod
102 + def _confidence(ptype, amount, locations, years, status, nm, low) -> float:
103 + c = 0.40
104 + if nm:
105 + c += 0.20
106 + if amount:
107 + c += 0.18
108 + if locations:
109 + c += 0.10
110 + if years:
111 + c += 0.06
112 + if status in ("planned", "in_progress", "completed"):
113 + c += 0.08
114 + # strong, specific phrasings
115 + if re.search(r"\b(announced|committed|invest\w+|will (build|open|invest))\b", low):
116 + c += 0.06
117 + # de-weight vague risk-factor / forward-looking hedges
118 + if re.search(r"\b(may|might|could|potential\w*|if we|uncertain)\b", low):
119 + c -= 0.08
120 + return max(0.0, min(1.0, c))
added code/audit/spid/taxonomy.py +261 −0
@@ -0,0 +1,261 @@
1 +# ============================================================
2 +# SPID — project taxonomy & lightweight value extractors
3 +#
4 +# Shared by both the rule-based and the LLM extractor so that
5 +# `project_type`, `technology`, money, location and dates are
6 +# normalised the same way regardless of backend.
7 +# ============================================================
8 +from __future__ import annotations
9 +
10 +import re
11 +
12 +# ----------------------------------------------------------------------
13 +# Canonical project types. `signals` are high-precision regex fragments
14 +# (already lower-cased) used by the rule extractor and as the controlled
15 +# vocabulary the LLM is asked to classify into.
16 +# ----------------------------------------------------------------------
17 +PROJECT_TYPES: dict[str, dict] = {
18 + "manufacturing_expansion": {
19 + "label": "Manufacturing / Capacity Expansion",
20 + "signals": [
21 + r"(expand|increas\w+|add\w*|ramp\w*|scal\w+|new|build\w*|construct\w*|open\w*)"
22 + r"[^.]{0,60}(manufactur\w+|production|plant|factory|facilit\w+|capacity|assembly|fab\b)",
23 + ],
24 + },
25 + "plant_construction": {
26 + "label": "Plant / Site Construction",
27 + "signals": [r"(construct\w*|build\w*|broke ground|commission\w*|new (plant|site|facility))"
28 + r"[^.]{0,60}(plant|facility|site|campus|complex|refinery|mill)"],
29 + },
30 + "data_center": {
31 + "label": "Data Center",
32 + "signals": [r"data\s?cent(er|re)s?", r"hyperscal\w+", r"server farm"],
33 + },
34 + "cloud_migration": {
35 + "label": "Cloud Migration",
36 + "signals": [r"(migrat\w+|mov\w+|transition\w*)[^.]{0,40}(cloud)",
37 + r"cloud (migration|platform|adoption|infrastructure|first)"],
38 + },
39 + "ai_initiative": {
40 + "label": "AI / ML Initiative",
41 + "signals": [r"\b(artificial intelligence|machine learning|generative ai|gen ?ai|deep learning|"
42 + r"large language model|\bllm\b|neural network)\b",
43 + r"\bai[- ](powered|driven|enabled|capabilit\w+|platform|model|tools?)\b"],
44 + },
45 + "digital_transformation": {
46 + "label": "Digital Transformation",
47 + "signals": [r"digital (transformation|transition|initiative|strateg\w+|platform)",
48 + r"digitali[sz]ation"],
49 + },
50 + "erp_implementation": {
51 + "label": "ERP / Systems Implementation",
52 + "signals": [r"\b(erp|sap|oracle|workday|s/?4 ?hana)\b[^.]{0,40}(implement\w+|deploy\w+|migrat\w+|system|platform)",
53 + r"(implement\w+|deploy\w+|upgrad\w+)[^.]{0,40}\berp\b",
54 + r"enterprise resource planning"],
55 + },
56 + "rd_program": {
57 + "label": "R&D Program",
58 + "signals": [r"research and development", r"\br&d\b",
59 + r"(develop\w+|invest\w+)[^.]{0,40}(next[- ]generation|new technolog\w+|new product)"],
60 + },
61 + "product_launch": {
62 + "label": "New Product / Launch",
63 + "signals": [r"(launch\w*|introduc\w+|releas\w+|roll\w*[- ]?out|unveil\w+)"
64 + r"[^.]{0,50}(product|platform|service|offering|solution|model|line)",
65 + r"new product (line|launch|introduction)"],
66 + },
67 + "drug_pipeline": {
68 + "label": "Drug / Clinical Pipeline",
69 + "signals": [r"(phase\s?(i{1,3}|1|2|3)\b)", r"clinical trial", r"\bfda (approval|clearance|submission)\b",
70 + r"(drug|therap\w+|candidate|molecule|pipeline)[^.]{0,40}(develop\w+|trial|approval)"],
71 + },
72 + "ma_integration": {
73 + "label": "M&A / Integration",
74 + "signals": [r"(acqui\w+|merg\w+|takeover|business combination)",
75 + r"integration of [a-z]", r"purchase of [a-z]"],
76 + },
77 + "geographic_expansion": {
78 + "label": "Geographic Expansion",
79 + "signals": [r"(expand\w*|enter\w*|launch\w*|establish\w*)[^.]{0,50}"
80 + r"(market|region|countr\w+|geograph\w+|international|overseas|abroad)",
81 + r"market entry"],
82 + },
83 + "capex_program": {
84 + "label": "Capital Expenditure Program",
85 + "signals": [r"capital (expenditure|expenditures|investment|spending|program|project|budget)",
86 + r"\bcapex\b", r"invest\w+[^.]{0,30}(\$|usd)"],
87 + },
88 + "supply_chain": {
89 + "label": "Supply Chain Initiative",
90 + "signals": [r"supply chain", r"(reshor\w+|nearshor\w+|onshor\w+|offshor\w+)",
91 + r"(distribution|logistics|fulfillment|warehouse|inventory)[^.]{0,40}"
92 + r"(network|center|optimi[sz]\w+|expansion|investment)"],
93 + },
94 + "automation": {
95 + "label": "Automation / Robotics",
96 + "signals": [r"automat\w+", r"robot\w+", r"(industrial|warehouse) automation"],
97 + },
98 + "sustainability": {
99 + "label": "Sustainability / ESG",
100 + "signals": [r"sustainabilit\w+", r"\besg\b", r"net[- ]zero", r"carbon (neutral|reduction|footprint|emissions?)",
101 + r"decarboni[sz]\w+", r"emission reduction"],
102 + },
103 + "energy_transition": {
104 + "label": "Energy Transition / Renewables",
105 + "signals": [r"renewable (energy|power)", r"(solar|wind|hydrogen|geothermal|battery storage)[^.]{0,40}"
106 + r"(project|facility|farm|plant|capacity|investment)",
107 + r"clean energy", r"energy transition"],
108 + },
109 + "infrastructure": {
110 + "label": "Infrastructure Investment",
111 + "signals": [r"infrastructure (investment|project|upgrade|expansion|spending|build\w*)",
112 + r"(network|grid|pipeline|broadband|fiber) (expansion|build\w*|upgrade|deployment)"],
113 + },
114 + "cost_reduction": {
115 + "label": "Cost Reduction / Restructuring",
116 + "signals": [r"(cost (reduction|savings|cutting)|restructur\w+|efficiency (program|initiative)|"
117 + r"productivity (program|initiative)|streamlin\w+|optimi[sz]ation (program|initiative))"],
118 + },
119 + "partnership": {
120 + "label": "Strategic Partnership",
121 + "signals": [r"(strategic (partnership|alliance|collaboration)|joint venture|\bjv\b|"
122 + r"(partner\w+|collaborat\w+|team\w* up)[^.]{0,40}(with))"],
123 + },
124 + "technology_deployment": {
125 + "label": "Technology Deployment",
126 + "signals": [r"(deploy\w+|implement\w+|roll\w*[- ]?out|adopt\w+|install\w+)[^.]{0,50}"
127 + r"(technolog\w+|system|platform|software|solution|tool)"],
128 + },
129 +}
130 +
131 +# ----------------------------------------------------------------------
132 +# Technology vocabulary -> canonical tag
133 +# ----------------------------------------------------------------------
134 +TECHNOLOGY_TERMS: dict[str, str] = {
135 + r"\b(artificial intelligence|machine learning|generative ai|gen ?ai|deep learning|llm|neural)\b": "AI/ML",
136 + r"\bcloud\b": "Cloud",
137 + r"\b(erp|sap|oracle|workday|s/?4 ?hana|peoplesoft)\b": "ERP",
138 + r"\b(robot\w+|automation|cobots?)\b": "Robotics/Automation",
139 + r"\b(semiconductor|chip|wafer|fab|node\b|nanometer|foundry)\b": "Semiconductors",
140 + r"\b(solar|wind|renewable|hydrogen|geothermal)\b": "Renewable Energy",
141 + r"\b(battery|batteries|energy storage|lithium)\b": "Battery/Storage",
142 + r"\b(ev|electric vehicle|charging)\b": "Electric Vehicles",
143 + r"\b(blockchain|crypto|digital asset|tokeni[sz])\b": "Blockchain",
144 + r"\b(5g|broadband|fiber|fibre|network infrastructure)\b": "Connectivity",
145 + r"\b(data\s?cent(er|re)|hyperscale|server)\b": "Data Center",
146 + r"\b(cyber ?security|encryption|zero trust)\b": "Cybersecurity",
147 + r"\b(iot|internet of things|sensors?|edge computing)\b": "IoT/Edge",
148 + r"\b(quantum)\b": "Quantum",
149 + r"\b(biolog\w+|gene|mrna|crispr|cell therapy|molecul\w+)\b": "Biotech",
150 + r"\b(saas|software[- ]as[- ]a[- ]service|platform)\b": "Software/SaaS",
151 +}
152 +
153 +# ----------------------------------------------------------------------
154 +# Status keywords -> canonical status
155 +# ----------------------------------------------------------------------
156 +STATUS_TERMS: list[tuple[str, str]] = [
157 + (r"\b(completed|commission\w+|operational|in service|came online|opened|finalized)\b", "completed"),
158 + (r"\b(under construction|underway|in progress|ongoing|ramping|being (built|implemented))\b", "in_progress"),
159 + (r"\b(announc\w+|plan\w+|intend\w+|expect\w+|will (build|invest|launch|open)|propos\w+|"
160 + r"anticipat\w+|target\w+|by (20\d\d)|slated)\b", "planned"),
161 +]
162 +
163 +# ----------------------------------------------------------------------
164 +# Geography
165 +# ----------------------------------------------------------------------
166 +US_STATES = {
167 + "alabama","alaska","arizona","arkansas","california","colorado","connecticut","delaware",
168 + "florida","georgia","hawaii","idaho","illinois","indiana","iowa","kansas","kentucky",
169 + "louisiana","maine","maryland","massachusetts","michigan","minnesota","mississippi",
170 + "missouri","montana","nebraska","nevada","new hampshire","new jersey","new mexico",
171 + "new york","north carolina","north dakota","ohio","oklahoma","oregon","pennsylvania",
172 + "rhode island","south carolina","south dakota","tennessee","texas","utah","vermont",
173 + "virginia","washington","west virginia","wisconsin","wyoming",
174 +}
175 +COUNTRIES = {
176 + "united states","u.s.","usa","canada","mexico","china","india","japan","germany","france",
177 + "united kingdom","u.k.","ireland","brazil","singapore","south korea","korea","taiwan",
178 + "vietnam","thailand","malaysia","indonesia","australia","spain","italy","netherlands",
179 + "switzerland","sweden","poland","israel","saudi arabia","united arab emirates","uae",
180 + "philippines","czech republic","hungary","turkey","egypt","south africa","argentina","chile",
181 +}
182 +
183 +_MONEY_RE = re.compile(
184 + r"(?:US)?\$\s?(\d[\d,]*\.?\d*)\s?(billion|bn|million|mm|mn|m\b|thousand|k\b|trillion)?",
185 + re.IGNORECASE,
186 +)
187 +_MULT = {
188 + "trillion": 1e12, "billion": 1e9, "bn": 1e9, "million": 1e6, "mm": 1e6, "mn": 1e6,
189 + "m": 1e6, "thousand": 1e3, "k": 1e3,
190 +}
191 +_YEAR_RE = re.compile(r"\b(?:fiscal\s+|fy\s?)?(20[0-3]\d|19[89]\d)\b", re.IGNORECASE)
192 +_QUARTER_RE = re.compile(r"\bQ([1-4])\s?(20\d\d)\b", re.IGNORECASE)
193 +_TECH_COMPILED = [(re.compile(p, re.IGNORECASE), tag) for p, tag in TECHNOLOGY_TERMS.items()]
194 +_STATUS_COMPILED = [(re.compile(p, re.IGNORECASE), s) for p, s in STATUS_TERMS]
195 +_CITY_STATE_RE = re.compile(r"\bin\s+([A-Z][a-zA-Z.]+(?:\s[A-Z][a-zA-Z.]+){0,2}),\s+([A-Z][a-zA-Z]+)")
196 +
197 +
198 +def extract_money(text: str) -> tuple[float | None, str | None]:
199 + """Return (usd_amount, raw_span) for the largest explicit dollar figure."""
200 + best_val, best_raw = None, None
201 + for m in _MONEY_RE.finditer(text):
202 + try:
203 + num = float(m.group(1).replace(",", ""))
204 + except ValueError:
205 + continue
206 + unit = (m.group(2) or "").lower().strip()
207 + val = num * _MULT.get(unit, 1.0)
208 + if best_val is None or val > best_val:
209 + best_val, best_raw = val, m.group(0).strip()
210 + return best_val, best_raw
211 +
212 +
213 +def extract_locations(text: str) -> list[str]:
214 + low = text.lower()
215 + found: list[str] = []
216 + for st in US_STATES:
217 + if re.search(rf"\b{re.escape(st)}\b", low):
218 + found.append(st.title())
219 + for co in COUNTRIES:
220 + if re.search(rf"\b{re.escape(co)}\b", low):
221 + found.append(co.upper() if len(co) <= 4 else co.title())
222 + for m in _CITY_STATE_RE.finditer(text):
223 + found.append(f"{m.group(1)}, {m.group(2)}")
224 + # de-dup preserving order
225 + seen, out = set(), []
226 + for f in found:
227 + k = f.lower()
228 + if k not in seen:
229 + seen.add(k); out.append(f)
230 + return out[:6]
231 +
232 +
233 +def extract_years(text: str) -> list[int]:
234 + yrs = {int(y) for y in _YEAR_RE.findall(text)}
235 + for m in _QUARTER_RE.finditer(text):
236 + yrs.add(int(m.group(2)))
237 + return sorted(yrs)
238 +
239 +
240 +def extract_technologies(text: str) -> list[str]:
241 + tags = []
242 + for rx, tag in _TECH_COMPILED:
243 + if rx.search(text) and tag not in tags:
244 + tags.append(tag)
245 + return tags
246 +
247 +
248 +def detect_status(text: str) -> str:
249 + for rx, status in _STATUS_COMPILED:
250 + if rx.search(text):
251 + return status
252 + return "mentioned"
253 +
254 +
255 +# pre-compile project-type signals
256 +TYPE_SIGNALS = {
257 + ptype: [re.compile(sig, re.IGNORECASE) for sig in spec["signals"]]
258 + for ptype, spec in PROJECT_TYPES.items()
259 +}
260 +TYPE_LABELS = {k: v["label"] for k, v in PROJECT_TYPES.items()}
261 +ALL_TYPES = list(PROJECT_TYPES.keys())
added code/audit/spid_snapshot/SHA256SUMS +9 −0
@@ -0,0 +1,9 @@
1 +4ba1811f9513ab472855764ba8d09363060fed6e7d009f68f5b2749e85833ab1 worktrees/FIRC-0001/code/audit/spid_snapshot/base.py
2 +8c4a273631e0015867a1c88d92e284faca906ea16e9af2595dfa5cb06d5b8d06 worktrees/FIRC-0001/code/audit/spid_snapshot/config.py
3 +c53536becfe273f03882c309ba6ac79d4423c5bf1320865d044a0d00effada90 worktrees/FIRC-0001/code/audit/spid_snapshot/ingest.py
4 +ae7f3f61a8de36842ba8c99a566883465fe8d6672ec723ee438f963e50d4eb89 worktrees/FIRC-0001/code/audit/spid_snapshot/llm.py
5 +6135f39adb9e78b7e50b81a9034db07e45f30fcb3a329df5c6c9a79bad028712 worktrees/FIRC-0001/code/audit/spid_snapshot/pipeline.py
6 +925ab1258b2a364f1c9ad5be5817b4d815fe11e7fca5ad32f3ac71b5be59652d worktrees/FIRC-0001/code/audit/spid_snapshot/resolve.py
7 +7f191c18b0426d83931fa6f1f461f3a2d43bcc1d151a14a6e084f24db6766807 worktrees/FIRC-0001/code/audit/spid_snapshot/rules.py
8 +1219c923fff72b9946bc068baacd70802fd5fab86766e62141dbdd2ea066a54f worktrees/FIRC-0001/code/audit/spid_snapshot/store.py
9 +92ad69bcc4e448d03496943221fdcb78b4326baa7db682a6b6713c260c9c4e7a worktrees/FIRC-0001/code/audit/spid_snapshot/taxonomy.py
added code/audit/spid_snapshot/base.py +69 −0
@@ -0,0 +1,69 @@
1 +# ============================================================
2 +# SPID — extraction interface & record schema
3 +# ============================================================
4 +from __future__ import annotations
5 +
6 +import hashlib
7 +from dataclasses import dataclass, field, asdict
8 +from typing import Protocol
9 +
10 +
11 +@dataclass
12 +class ProjectMention:
13 + """A single project signal extracted from one filing section.
14 +
15 + Mentions are the atomic unit; `resolve.py` merges many mentions
16 + (across filings & years) into a single Project with a timeline.
17 + """
18 + mention_id: str
19 + accession_number: str
20 + cik: str
21 + section_id: str
22 + section_name: str
23 +
24 + project_name: str
25 + project_type: str
26 + description: str
27 + objective: str = ""
28 +
29 + amount_usd: float | None = None
30 + amount_raw: str | None = None
31 + locations: list[str] = field(default_factory=list)
32 + technologies: list[str] = field(default_factory=list)
33 + partners: list[str] = field(default_factory=list)
34 + suppliers: list[str] = field(default_factory=list)
35 + benefits: list[str] = field(default_factory=list)
36 + risks: list[str] = field(default_factory=list)
37 + years: list[int] = field(default_factory=list)
38 + status: str = "mentioned"
39 +
40 + confidence: float = 0.0
41 + backend: str = "rules"
42 +
43 + def to_row(self) -> dict:
44 + d = asdict(self)
45 + # store list fields as JSON-friendly via duckdb list[str]
46 + return d
47 +
48 + @staticmethod
49 + def make_id(accession: str, section_id: str, ptype: str, anchor: str) -> str:
50 + h = hashlib.sha1(f"{accession}|{section_id}|{ptype}|{anchor}".encode()).hexdigest()
51 + return h[:20]
52 +
53 +
54 +class Extractor(Protocol):
55 + name: str
56 +
57 + def extract(self, *, text: str, accession_number: str, cik: str,
58 + section_id: str, section_name: str) -> list[ProjectMention]:
59 + ...
60 +
61 +
62 +def get_extractor(backend: str):
63 + """Factory. `backend` in {'rules','llm'}."""
64 + backend = (backend or "rules").lower()
65 + if backend == "llm":
66 + from spid.extract.llm import LLMExtractor
67 + return LLMExtractor()
68 + from spid.extract.rules import RuleExtractor
69 + return RuleExtractor()
added code/audit/spid_snapshot/config.py +126 −0
@@ -0,0 +1,126 @@
1 +# ============================================================
2 +# SPID — configuration & paths
3 +# ============================================================
4 +from __future__ import annotations
5 +
6 +import logging
7 +import os
8 +from dataclasses import dataclass, field
9 +from functools import lru_cache
10 +from pathlib import Path
11 +
12 +# project-db/ (this file lives in project-db/spid/config.py)
13 +PROJECT_ROOT = Path(__file__).resolve().parent.parent
14 +
15 +
16 +def _load_dotenv() -> None:
17 + """Minimal .env loader (no python-dotenv dependency required)."""
18 + env_path = PROJECT_ROOT / ".env"
19 + if not env_path.exists():
20 + return
21 + for line in env_path.read_text().splitlines():
22 + line = line.strip()
23 + if not line or line.startswith("#") or "=" not in line:
24 + continue
25 + key, _, val = line.partition("=")
26 + key, val = key.strip(), val.strip().strip('"').strip("'")
27 + os.environ.setdefault(key, val)
28 +
29 +
30 +_load_dotenv()
31 +# The parent UQO_RESEARCH_DB that holds the parsed corpus.
32 +RESEARCH_ROOT = PROJECT_ROOT.parent
33 +SOURCE_DATA = RESEARCH_ROOT / "data"
34 +
35 +
36 +def _parse_scope(spec: str) -> dict:
37 + out: dict = {}
38 + for part in (spec or "").split(";"):
39 + part = part.strip()
40 + if not part or ":" not in part:
41 + continue
42 + form, items = part.split(":", 1)
43 + out[form.strip()] = [i.strip() for i in items.split(",") if i.strip()]
44 + return out
45 +
46 +
47 +@dataclass(frozen=True)
48 +class Settings:
49 + """Resolved runtime configuration for the SPID platform."""
50 +
51 + # --- source corpus (read-only) -------------------------------------
52 + source_data: Path = SOURCE_DATA
53 + parquet_dir: Path = SOURCE_DATA / "parquet"
54 + processed_dir: Path = SOURCE_DATA / "processed"
55 +
56 + @property
57 + def filings_parquet(self) -> Path:
58 + return self.parquet_dir / "filings" / "filings.parquet"
59 +
60 + @property
61 + def sections_parquet(self) -> Path:
62 + return self.parquet_dir / "filing_sections" / "filing_sections.parquet"
63 +
64 + @property
65 + def companies_parquet(self) -> Path:
66 + return self.parquet_dir / "companies" / "companies.parquet"
67 +
68 + # --- SPID outputs (writable) ---------------------------------------
69 + db_path: Path = PROJECT_ROOT / "data" / "db" / "spid.duckdb"
70 + out_dir: Path = PROJECT_ROOT / "data"
71 +
72 + # --- extraction ----------------------------------------------------
73 + backend: str = field(
74 + default_factory=lambda: os.environ.get("SPID_BACKEND", "llm")
75 + ) # "llm" (gpt-4o-mini) | "rules"
76 + min_confidence: float = 0.45
77 + # which parsed sections feed the extractor, per form (None/empty = every section)
78 + section_scope: dict = field(default_factory=lambda: _parse_scope(os.environ.get(
79 + "SPID_SECTION_SCOPE",
80 + "10-K:business,risk_factors,properties,md_and_a,quantitative_disclosures")))
81 + max_section_chars: int = 600_000
82 +
83 + # --- embeddings ----------------------------------------------------
84 + embedding_model: str = field(
85 + default_factory=lambda: os.environ.get("SPID_EMBED_MODEL", "all-MiniLM-L6-v2")
86 + )
87 +
88 + # --- LLM backend (OpenAI-compatible) -------------------------------
89 + llm_base_url: str = field(
90 + default_factory=lambda: os.environ.get(
91 + "SPID_LLM_BASE_URL", "https://api.openai.com/v1"
92 + )
93 + )
94 + llm_model: str = field(
95 + default_factory=lambda: os.environ.get("SPID_LLM_MODEL", "gpt-4o-mini")
96 + )
97 + llm_api_key: str = field(
98 + default_factory=lambda: os.environ.get("OPENAI_API_KEY", "")
99 + or os.environ.get("MACLUSTR_API_KEY", "")
100 + )
101 + llm_timeout: float = 120.0
102 +
103 + def ensure_dirs(self) -> None:
104 + self.db_path.parent.mkdir(parents=True, exist_ok=True)
105 + (self.out_dir / "exports").mkdir(parents=True, exist_ok=True)
106 +
107 +
108 +@lru_cache(maxsize=1)
109 +def get_settings() -> Settings:
110 + s = Settings()
111 + s.ensure_dirs()
112 + return s
113 +
114 +
115 +def get_logger(name: str) -> logging.Logger:
116 + logger = logging.getLogger(f"spid.{name}")
117 + if not logger.handlers:
118 + h = logging.StreamHandler()
119 + h.setFormatter(
120 + logging.Formatter("%(asctime)s %(levelname)-7s %(name)s | %(message)s",
121 + datefmt="%H:%M:%S")
122 + )
123 + logger.addHandler(h)
124 + logger.setLevel(os.environ.get("SPID_LOG_LEVEL", "INFO"))
125 + logger.propagate = False
126 + return logger
added code/audit/spid_snapshot/ingest.py +135 −0
@@ -0,0 +1,135 @@
1 +# ============================================================
2 +# SPID — raw 10-K ingester
3 +#
4 +# The pre-parsed corpus only sectioned 10-Q & 8-K filings; the
5 +# 10-K HTML was left raw. This module cleans the raw 10-K HTML
6 +# and extracts the project-dense Items (1 Business, 1A Risk,
7 +# 7 MD&A, 2 Properties) into a SPID-owned `spid_sections` table
8 +# that `v_sections` unions in — so 10-Ks flow through the same
9 +# extraction pipeline.
10 +# ============================================================
11 +from __future__ import annotations
12 +
13 +import glob
14 +import re
15 +import uuid
16 +from pathlib import Path
17 +
18 +from spid.config import get_logger, get_settings
19 +from spid.store import Store
20 +
21 +log = get_logger("ingest")
22 +
23 +RAW_SEC = get_settings().source_data / "raw" / "sec"
24 +
25 +# 10-K items we keep (where projects/capex/strategy concentrate)
26 +_KEEP_ITEMS = {
27 + "1": "business", "1A": "risk_factors", "2": "properties",
28 + "7": "md_and_a", "7A": "quantitative_disclosures",
29 +}
30 +_ITEM_RE = re.compile(r"(?:^|\n)\s*ITEM\s+(\d+[AB]?)\s*[.\-:—–]", re.IGNORECASE)
31 +_TAG_RE = re.compile(r"<[^>]+>")
32 +_WS_RE = re.compile(r"[ \t]+")
33 +_NL_RE = re.compile(r"\n{3,}")
34 +
35 +
36 +def _clean_html(path: Path) -> str:
37 + raw = path.read_text(errors="ignore")
38 + try:
39 + from bs4 import BeautifulSoup
40 + soup = BeautifulSoup(raw, "lxml")
41 + for t in soup(["script", "style", "table"]):
42 + t.decompose()
43 + text = soup.get_text("\n")
44 + except Exception: # noqa: BLE001
45 + text = _TAG_RE.sub(" ", raw)
46 + import html as _html
47 + text = _html.unescape(text)
48 + text = _WS_RE.sub(" ", text)
49 + text = _NL_RE.sub("\n\n", text)
50 + return text
51 +
52 +
53 +def _extract_items(text: str) -> list[tuple[str, str]]:
54 + matches = list(_ITEM_RE.finditer(text))
55 + if not matches:
56 + return [("full_body", text)]
57 + out: list[tuple[str, str]] = []
58 + for i, m in enumerate(matches):
59 + item = m.group(1).upper()
60 + if item not in _KEEP_ITEMS:
61 + continue
62 + start = m.start()
63 + end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
64 + body = text[start:end].strip()
65 + if len(body) > 400: # skip the table-of-contents stubs
66 + out.append((_KEEP_ITEMS[item], body[:600_000]))
67 + # keep the longest instance of each section (TOC vs real body)
68 + best: dict[str, str] = {}
69 + for name, body in out:
70 + if name not in best or len(body) > len(best[name]):
71 + best[name] = body
72 + return list(best.items())
73 +
74 +
75 +def _find_primary(acc: str, cik: str) -> Path | None:
76 + d = RAW_SEC / cik / "10-K" / acc.replace("-", "")
77 + if not d.is_dir():
78 + return None
79 + cands = glob.glob(str(d / "*.htm")) + glob.glob(str(d / "*.html")) + glob.glob(str(d / "*.txt"))
80 + cands = [c for c in cands if "R" + "" not in Path(c).name or True]
81 + if not cands:
82 + return None
83 + return Path(max(cands, key=lambda p: Path(p).stat().st_size))
84 +
85 +
86 +_SPID_SECTIONS_DDL = """
87 +CREATE TABLE IF NOT EXISTS spid_sections (
88 + section_id VARCHAR PRIMARY KEY,
89 + accession_number VARCHAR,
90 + section_name VARCHAR,
91 + item_number VARCHAR,
92 + text VARCHAR,
93 + word_count INTEGER
94 +);
95 +"""
96 +
97 +
98 +def ingest_10k(store: Store, *, ciks: list[str] | None = None, limit: int | None = None) -> dict:
99 + con = store.con
100 + con.execute(_SPID_SECTIONS_DDL)
101 + where = "form_type='10-K'"
102 + if ciks:
103 + q = ",".join("'" + c + "'" for c in ciks)
104 + where += f" AND cik IN ({q})"
105 + sql = f"SELECT accession_number, cik FROM src_filings WHERE {where} ORDER BY filing_date DESC"
106 + if limit:
107 + sql += f" LIMIT {int(limit)}"
108 + filings = con.execute(sql).fetchall()
109 +
110 + rows, done, ok = [], 0, 0
111 + for acc, cik in filings:
112 + done += 1
113 + # skip if already ingested
114 + if con.execute("SELECT 1 FROM spid_sections WHERE accession_number=? LIMIT 1", [acc]).fetchone():
115 + ok += 1
116 + continue
117 + path = _find_primary(acc, cik)
118 + if not path:
119 + continue
120 + try:
121 + text = _clean_html(path)
122 + for name, body in _extract_items(text):
123 + rows.append((str(uuid.uuid4()), acc, name, "", body, len(body.split())))
124 + ok += 1
125 + except Exception as e: # noqa: BLE001
126 + log.warning("ingest failed %s: %s", acc, e)
127 + if len(rows) >= 400:
128 + con.executemany("INSERT OR REPLACE INTO spid_sections VALUES (?,?,?,?,?,?)", rows); rows = []
129 + if done % 50 == 0:
130 + log.info(" ... ingested %d/%d 10-Ks", done, len(filings))
131 + if rows:
132 + con.executemany("INSERT OR REPLACE INTO spid_sections VALUES (?,?,?,?,?,?)", rows)
133 + n = store.count("spid_sections")
134 + log.info("Ingested 10-K sections: %d filings processed, %d section rows total.", ok, n)
135 + return {"filings_processed": ok, "section_rows": n}
added code/audit/spid_snapshot/llm.py +202 −0
@@ -0,0 +1,202 @@
1 +# ============================================================
2 +# SPID — LLM extractor (OpenAI-compatible; default gpt-4o-mini)
3 +#
4 +# Uses the chat-completions JSON mode to extract structured
5 +# project records. Cost-controlled: only project-relevant
6 +# windows of each section are sent to the model.
7 +# ============================================================
8 +from __future__ import annotations
9 +
10 +import json
11 +import re
12 +import time
13 +
14 +import httpx
15 +
16 +from spid import taxonomy as tx
17 +from spid.config import get_logger, get_settings
18 +from spid.extract.base import ProjectMention
19 +from spid.extract.rules import _PROJECT_GATE
20 +
21 +log = get_logger("extract.llm")
22 +
23 +_SYSTEM = """You are a financial analyst extracting CORPORATE PROJECTS from SEC filing text.
24 +A "project" is any concrete initiative, investment, program, expansion, construction,
25 +product/R&D effort, acquisition/integration, technology deployment, partnership, or
26 +strategic program the company is pursuing.
27 +
28 +Return STRICT JSON: {"projects": [ ... ]}. Each project object has:
29 + project_name : short descriptive name (use the official name if one is stated)
30 + project_type : EXACTLY ONE of the allowed types below
31 + description : 1 sentence, grounded in the text (no speculation)
32 + objective : the goal/why (or "")
33 + status : one of planned | in_progress | completed | mentioned
34 + amount_usd : numeric USD figure if stated (e.g. 1500000000), else null
35 + locations : array of places (city/state/country), else []
36 + technologies : array of technologies involved, else []
37 + partners : array of named partners/JV counterparties, else []
38 + benefits : array of expected benefits, else []
39 + risks : array of stated risks, else []
40 + years : array of 4-digit years mentioned, else []
41 + confidence : 0.0-1.0 (how clearly this is a real, concrete project)
42 +
43 +Allowed project_type values:
44 +%s
45 +
46 +Rules:
47 +- Extract ONLY real initiatives, not generic risk-factor boilerplate or hypotheticals.
48 +- If the text describes no concrete project, return {"projects": []}.
49 +- Do not invent facts. Numbers/locations must appear in the text.
50 +- Prefer fewer high-quality projects over many vague ones.""" % "\n".join(
51 + f" - {k}: {v}" for k, v in tx.TYPE_LABELS.items()
52 +)
53 +
54 +
55 +def _focus(text: str, budget: int = 9000) -> str:
56 + """Keep only windows around project-signal hits to bound token cost."""
57 + if len(text) <= budget:
58 + return text
59 + spans: list[tuple[int, int]] = []
60 + for m in _PROJECT_GATE.finditer(text):
61 + spans.append((max(0, m.start() - 220), min(len(text), m.end() + 380)))
62 + if sum(b - a for a, b in spans) > budget * 1.4:
63 + break
64 + if not spans:
65 + return text[:budget]
66 + # merge overlapping windows
67 + spans.sort()
68 + merged = [spans[0]]
69 + for a, b in spans[1:]:
70 + if a <= merged[-1][1]:
71 + merged[-1] = (merged[-1][0], max(merged[-1][1], b))
72 + else:
73 + merged.append((a, b))
74 + chunks, total = [], 0
75 + for a, b in merged:
76 + chunks.append(text[a:b])
77 + total += b - a
78 + if total >= budget:
79 + break
80 + return "\n...\n".join(chunks)
81 +
82 +
83 +class LLMExtractor:
84 + name = "llm"
85 +
86 + def __init__(self) -> None:
87 + self.s = get_settings()
88 + if not self.s.llm_api_key:
89 + raise RuntimeError("No LLM API key configured (OPENAI_API_KEY / MACLUSTR_API_KEY).")
90 + self._client = httpx.Client(
91 + base_url=self.s.llm_base_url,
92 + headers={"Authorization": f"Bearer {self.s.llm_api_key}"},
93 + timeout=self.s.llm_timeout,
94 + )
95 +
96 + def _call(self, content: str, retries: int = 3) -> dict:
97 + payload = {
98 + "model": self.s.llm_model,
99 + "messages": [
100 + {"role": "system", "content": _SYSTEM},
101 + {"role": "user", "content": content},
102 + ],
103 + "temperature": 0,
104 + "response_format": {"type": "json_object"},
105 + "max_tokens": 2000,
106 + }
107 + last = None
108 + for attempt in range(retries):
109 + try:
110 + r = self._client.post("/chat/completions", json=payload)
111 + if r.status_code == 429 or r.status_code >= 500:
112 + raise httpx.HTTPStatusError("retryable", request=r.request, response=r)
113 + r.raise_for_status()
114 + txt = r.json()["choices"][0]["message"]["content"]
115 + return json.loads(txt)
116 + except (httpx.HTTPError, json.JSONDecodeError, KeyError) as e:
117 + last = e
118 + time.sleep(1.5 * (attempt + 1))
119 + log.warning("LLM call failed after %d retries: %s", retries, last)
120 + return {"projects": []}
121 +
122 + def extract(self, *, text: str, accession_number: str, cik: str,
123 + section_id: str, section_name: str) -> list[ProjectMention]:
124 + if not text or not _PROJECT_GATE.search(text):
125 + return []
126 + content = (f"SECTION: {section_name}\n\n" + _focus(text))[:14000]
127 + data = self._call(content)
128 + out: list[ProjectMention] = []
129 + for p in data.get("projects", []) or []:
130 + if not isinstance(p, dict):
131 + continue
132 + ptype = self._norm_type(p.get("project_type", ""))
133 + desc = str(p.get("description", "")).strip()[:600]
134 + if not desc:
135 + continue
136 + amount = p.get("amount_usd")
137 + try:
138 + amount = float(amount) if amount not in (None, "", "null") else None
139 + except (TypeError, ValueError):
140 + amount = None
141 + anchor = f"{ptype}:{str(p.get('project_name',''))[:40]}"
142 + conf = p.get("confidence", 0.6)
143 + try:
144 + conf = float(conf)
145 + except (TypeError, ValueError):
146 + conf = 0.6
147 + out.append(ProjectMention(
148 + mention_id=ProjectMention.make_id(accession_number, section_id, ptype, anchor),
149 + accession_number=accession_number, cik=cik,
150 + section_id=section_id, section_name=section_name,
151 + project_name=str(p.get("project_name") or tx.TYPE_LABELS.get(ptype, ptype))[:200],
152 + project_type=ptype, description=desc,
153 + objective=str(p.get("objective", ""))[:400],
154 + amount_usd=amount,
155 + amount_raw=(f"${amount:,.0f}" if amount else None),
156 + locations=_strlist(p.get("locations")),
157 + technologies=_strlist(p.get("technologies")) or tx.extract_technologies(desc),
158 + partners=_strlist(p.get("partners")),
159 + benefits=_strlist(p.get("benefits")),
160 + risks=_strlist(p.get("risks")),
161 + years=_intlist(p.get("years")) or tx.extract_years(desc),
162 + status=self._norm_status(p.get("status")),
163 + confidence=round(max(0.0, min(1.0, conf)), 3),
164 + backend=self.name,
165 + ))
166 + return out
167 +
168 + @staticmethod
169 + def _norm_type(raw: str) -> str:
170 + raw = (raw or "").strip().lower().replace(" ", "_").replace("-", "_")
171 + if raw in tx.PROJECT_TYPES:
172 + return raw
173 + # fuzzy contains match against labels/keys
174 + for k, label in tx.TYPE_LABELS.items():
175 + if raw and (raw in k or raw in label.lower().replace(" ", "_")):
176 + return k
177 + return "technology_deployment"
178 +
179 + @staticmethod
180 + def _norm_status(raw) -> str:
181 + raw = (str(raw or "")).strip().lower()
182 + return raw if raw in ("planned", "in_progress", "completed", "mentioned") else "mentioned"
183 +
184 +
185 +def _strlist(v) -> list[str]:
186 + if not v:
187 + return []
188 + if isinstance(v, str):
189 + v = [v]
190 + return [str(x).strip()[:120] for x in v if str(x).strip()][:8]
191 +
192 +
193 +def _intlist(v) -> list[int]:
194 + out = []
195 + for x in (v or []):
196 + try:
197 + y = int(str(x)[:4])
198 + if 1980 <= y <= 2040:
199 + out.append(y)
200 + except (TypeError, ValueError):
201 + continue
202 + return sorted(set(out))
added code/audit/spid_snapshot/pipeline.py +95 −0
@@ -0,0 +1,95 @@
1 +# ============================================================
2 +# SPID — extraction engine
3 +#
4 +# Iterates enriched sections, runs the chosen extractor
5 +# (rules or gpt-4o-mini), filters by confidence, and loads
6 +# mentions into DuckDB. LLM calls are threaded (IO-bound).
7 +# ============================================================
8 +from __future__ import annotations
9 +
10 +from concurrent.futures import ThreadPoolExecutor, as_completed
11 +
12 +from spid.config import get_logger, get_settings
13 +from spid.extract.base import get_extractor
14 +from spid.store import Store
15 +
16 +log = get_logger("pipeline")
17 +
18 +
19 +def _ctx_from_rows(rows: list[dict]) -> dict:
20 + ctx = {}
21 + for r in rows:
22 + ctx.setdefault(r["accession_number"], {
23 + "ticker": r.get("ticker"),
24 + "company_name": r.get("company_name") or r.get("filing_company"),
25 + "filing_company": r.get("filing_company"),
26 + "form_type": r.get("form_type"),
27 + "filing_date": r.get("filing_date"),
28 + "fiscal_year": r.get("fiscal_year"),
29 + "sector": r.get("sector"),
30 + })
31 + return ctx
32 +
33 +
34 +def run_extraction(store: Store, *, backend: str | None = None,
35 + ciks: list[str] | None = None, forms: list[str] | None = None,
36 + limit_sections: int | None = None, min_words: int = 30,
37 + workers: int = 8, min_confidence: float | None = None,
38 + flush_every: int = 500) -> dict:
39 + s = get_settings()
40 + backend = backend or s.backend
41 + min_conf = s.min_confidence if min_confidence is None else min_confidence
42 + extractor = get_extractor(backend)
43 +
44 + rows = store.sections_for(ciks=ciks, forms=forms, limit=limit_sections, min_words=min_words)
45 + ctx = _ctx_from_rows(rows)
46 + log.info("Extracting from %d sections | backend=%s | workers=%d", len(rows), backend, workers)
47 +
48 + kept = 0
49 + buf = []
50 + done_rows: list[tuple] = []
51 +
52 + def work(r: dict):
53 + return extractor.extract(text=r["text"], accession_number=r["accession_number"],
54 + cik=r["cik"], section_id=r["section_id"],
55 + section_name=r["section_name"])
56 +
57 + def flush():
58 + nonlocal kept, buf, done_rows
59 + if buf:
60 + kept += store.upsert_mentions(buf, ctx)
61 + buf = []
62 + if done_rows:
63 + store.mark_extracted(done_rows)
64 + done_rows = []
65 +
66 + done = 0
67 + if backend == "llm" and workers > 1:
68 + with ThreadPoolExecutor(max_workers=workers) as ex:
69 + futs = {ex.submit(work, r): r for r in rows}
70 + for fut in as_completed(futs):
71 + done += 1
72 + r = futs[fut]
73 + try:
74 + ms = [m for m in fut.result() if m.confidence >= min_conf]
75 + done_rows.append((r["section_id"], r["accession_number"], backend, len(ms)))
76 + except Exception as e: # noqa: BLE001
77 + log.warning("section failed: %s", e); ms = []
78 + buf.extend(ms)
79 + if len(buf) >= flush_every:
80 + flush()
81 + if done % 200 == 0:
82 + log.info(" ... %d/%d sections, %d mentions kept", done, len(rows), kept + len(buf))
83 + else:
84 + for r in rows:
85 + done += 1
86 + ms = [m for m in work(r) if m.confidence >= min_conf]
87 + done_rows.append((r["section_id"], r["accession_number"], backend, len(ms)))
88 + buf.extend(ms)
89 + if len(buf) >= flush_every:
90 + flush()
91 + if done % 2000 == 0:
92 + log.info(" ... %d/%d sections, %d mentions kept", done, len(rows), kept + len(buf))
93 + flush()
94 + log.info("Extraction complete: %d mentions from %d sections.", kept, len(rows))
95 + return {"sections": len(rows), "mentions": kept, "backend": backend}
added code/audit/spid_snapshot/resolve.py +75 −0
Large diff (75 lines) is not loaded by default to keep the page fast.Load diff
added code/audit/spid_snapshot/rules.py +120 −0
Large diff (120 lines) is not loaded by default to keep the page fast.Load diff
added code/audit/spid_snapshot/store.py +239 −0
Large diff (239 lines) is not loaded by default to keep the page fast.Load diff
added code/audit/spid_snapshot/taxonomy.py +261 −0
Large diff (261 lines) is not loaded by default to keep the page fast.Load diff
added results/n0_probe/manifest.json +27 −0
Large diff (28 lines) is not loaded by default to keep the page fast.Load diff
added results/n0_probe/schema.json +937 −0
Large diff (939 lines) is not loaded by default to keep the page fast.Load diff
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere