SPB Git
spboucher.ai

spb/firc0001_uqo

Public

N0: audit infrastructure (read-only helpers, hash-verified 2324b58 snapshot, spid import shim, schema probe)

Stage D node batch N0 for FIRC-0001. sha256 of the nine snapshot files equals the node copies
(extract/llm.py, rules.py, base.py, taxonomy.py, resolve.py mtime 2026-06-17); both DuckDB files
open read_only=True; TEMP tables allowed; DuckDB temp_directory redirected to ~/firc0001_review/tmp.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Nov 5, 2026) parent 678d233 Browse files

23 changed files +3,242 −0

23 changed files
Unified Split
added code/audit/a00_probe_schema.py +27 −0
@@ -0,0 +1,27 @@
1+"""N0 probe: open both DuckDB files read-only, describe the tables used by N1-N13."""
2+import sys, json
3+from pathlib import Path
4+sys.path.insert(0, str(Path(__file__).resolve().parent))
5+from common import *
6+
7+r = Run("n0_probe")
8+con = connect(SPID_DB)
9+out = {}
10+for t in ["extracted_sections", "project_mentions", "projects", "project_timeline", "spid_sections", "v_sections", "src_filings", "src_sections", "src_companies"]:
11+ out[t] = [tuple(x[:2]) for x in con.execute(f"DESCRIBE {t}").fetchall()]
12+out["views"] = con.execute("select view_name, sql from duckdb_views() where not internal").fetchall()
13+out["counts"] = {t: con.execute(f"select count(*) from {t}").fetchone()[0] for t in ["extracted_sections", "project_mentions", "projects", "project_timeline", "spid_sections"]}
14+out["ledger_words"] = con.execute("select count(*), sum(v.word_count) from extracted_sections e join (select section_id, word_count from src_sections union all select section_id, word_count from spid_sections) v using(section_id)").fetchone()
15+out["parquet_files"] = con.execute("select count(*), sum(length(text))/1e9 from src_sections").fetchone()
16+e = connect(EDGAR_DB)
17+for t in ["filings", "filing_sections", "companies", "annual_panel"]:
18+ out["edgar_"+t] = [tuple(x[:2]) for x in e.execute(f"DESCRIBE {t}").fetchall()]
19+out["edgar_tables"] = [x[0] for x in e.execute("select table_name from duckdb_tables() where not internal").fetchall()]
20+r.write_json("schema.json", out)
21+# can we create a temp table on a read_only connection?
22+try:
23+ con.execute("CREATE TEMP TABLE _t AS SELECT 1 AS x"); out["temp_ok"] = True
24+except Exception as ex:
25+ out["temp_ok"] = str(ex)
26+r.write_json("schema.json", out)
27+r.finish()
added code/audit/common.py +116 −0
@@ -0,0 +1,116 @@
1+"""Shared helpers for the FIRC-0001 Stage D read-only audits (run on M3U96b).
2+
3+Every script opens the two DuckDB files with read_only=True, SET threads=4,
4+SET memory_limit='24GB', writes CSV/JSON under ~/firc0001_review/results/<name>/
5+and a manifest.json (script sha256, DuckDB mtimes, start/end, row counts).
6+Nothing is ever written under ~/UQO_RESEARCH_DB.
7+"""
8+from __future__ import annotations
9+
10+import csv
11+import datetime as dt
12+import hashlib
13+import json
14+import os
15+import sys
16+import time
17+from pathlib import Path
18+
19+import duckdb
20+
21+HOME = Path(os.environ.get("HOME", "~")).expanduser()
22+RDB = HOME / "UQO_RESEARCH_DB"
23+SPID_DB = RDB / "project-db" / "data" / "db" / "spid.duckdb"
24+EDGAR_DB = RDB / "data" / "duckdb" / "edgar_research.duckdb"
25+WORK = HOME / "firc0001_review"
26+RESULTS = WORK / "results"
27+THREADS = int(os.environ.get("FIRC_THREADS", "4"))
28+MEMLIMIT = os.environ.get("FIRC_MEMLIMIT", "24GB")
29+SEED = 20261105
30+
31+
32+def connect(path: Path = SPID_DB) -> duckdb.DuckDBPyConnection:
33+ con = duckdb.connect(str(path), read_only=True)
34+ con.execute(f"SET threads={THREADS}")
35+ con.execute(f"SET memory_limit='{MEMLIMIT}'")
36+ con.execute("SET preserve_insertion_order=false")
37+ # spill files must never land next to the databases (default = <db>.tmp)
38+ (WORK / "tmp").mkdir(parents=True, exist_ok=True)
39+ con.execute(f"SET temp_directory='{WORK / 'tmp'}'")
40+ return con
41+
42+
43+def attach_edgar(con: duckdb.DuckDBPyConnection, alias: str = "edgar") -> None:
44+ con.execute(f"ATTACH '{EDGAR_DB}' AS {alias} (READ_ONLY)")
45+
46+
47+def sha256_file(p: Path) -> str:
48+ h = hashlib.sha256()
49+ with open(p, "rb") as f:
50+ for chunk in iter(lambda: f.read(1 << 20), b""):
51+ h.update(chunk)
52+ return h.hexdigest()
53+
54+
55+class Run:
56+ """Context for one audit script: output dir, timing, manifest."""
57+
58+ def __init__(self, name: str, script: str | None = None, out: str | None = None):
59+ self.name = name
60+ self.script = Path(script or sys.argv[0]).resolve()
61+ self.out = Path(out) if out else RESULTS / name
62+ self.out.mkdir(parents=True, exist_ok=True)
63+ self.t0 = time.time()
64+ self.started = dt.datetime.now().isoformat(timespec="seconds")
65+ self.rows: dict[str, int] = {}
66+ self.notes: dict = {}
67+
68+ def write_csv(self, fname: str, cols: list[str], rows) -> int:
69+ rows = list(rows)
70+ with open(self.out / fname, "w", newline="") as f:
71+ w = csv.writer(f)
72+ w.writerow(cols)
73+ w.writerows(rows)
74+ self.rows[fname] = len(rows)
75+ print(f"[{self.name}] {fname}: {len(rows)} rows", flush=True)
76+ return len(rows)
77+
78+ def dump(self, fname: str, con, sql: str, params=None):
79+ cur = con.execute(sql, params) if params else con.execute(sql)
80+ cols = [d[0] for d in cur.description]
81+ return self.write_csv(fname, cols, cur.fetchall())
82+
83+ def write_json(self, fname: str, obj) -> None:
84+ with open(self.out / fname, "w") as f:
85+ json.dump(obj, f, indent=2, default=str)
86+ print(f"[{self.name}] {fname} written", flush=True)
87+
88+ def finish(self, extra: dict | None = None) -> None:
89+ man = {
90+ "package": self.name,
91+ "script": str(self.script),
92+ "script_sha256": sha256_file(self.script) if self.script.exists() else None,
93+ "started": self.started,
94+ "ended": dt.datetime.now().isoformat(timespec="seconds"),
95+ "wall_seconds": round(time.time() - self.t0, 1),
96+ "node": os.uname().nodename,
97+ "python": sys.version.split()[0],
98+ "duckdb": duckdb.__version__,
99+ "threads": THREADS,
100+ "memory_limit": MEMLIMIT,
101+ "seed": SEED,
102+ "databases": {
103+ str(SPID_DB): {"bytes": SPID_DB.stat().st_size,
104+ "mtime": dt.datetime.fromtimestamp(SPID_DB.stat().st_mtime).isoformat()},
105+ str(EDGAR_DB): {"bytes": EDGAR_DB.stat().st_size,
106+ "mtime": dt.datetime.fromtimestamp(EDGAR_DB.stat().st_mtime).isoformat()},
107+ },
108+ "read_only": True,
109+ "row_counts": self.rows,
110+ "notes": self.notes,
111+ }
112+ if extra:
113+ man.update(extra)
114+ with open(self.out / "manifest.json", "w") as f:
115+ json.dump(man, f, indent=2, default=str)
116+ print(f"[{self.name}] done in {man['wall_seconds']} s", flush=True)
added code/audit/run_all.sh +12 −0
@@ -0,0 +1,12 @@
1+#!/bin/bash
2+# Stage D node batch for FIRC-0001: runs N1..N13 in the plan's order, sequentially, read-only.
3+# usage: bash run_all.sh (on M3U96b, from ~/firc0001_review; scripts/ holds the aNN_*.py copies)
4+set -u
5+cd "$HOME/firc0001_review"
6+ORDER="a01_ledger_reconciliation.py a03_resolver_speccurve.py a04_runs_segmenters.py a04_git_evidence.sh a02_grounding.py a05_focus_coverage.py a06_false_positive_classes.py a09_timeseries_cohort.py a10_universe_artefacts.py a07_8k_item_benchmark.py a08_rules_baseline.py a11_annotation_frame.py a12_env_lock.sh a13_export_stats_check.sh"
7+for S in $ORDER; do
8+ bash scripts/run_one.sh "$S"
9+ NAME=$(basename "$S" | sed -E 's/\.(py|sh)$//')
10+ while [ ! -f "logs/$NAME.done" ]; do sleep 60; done
11+ cat "logs/$NAME.done"
12+done
added code/audit/run_one.sh +13 −0
@@ -0,0 +1,13 @@
1+#!/bin/bash
2+# Run one audit script on M3U96b in the background under nice, with a log and a done-marker.
3+# usage: run_one.sh <script.py|script.sh> [args...] (from ~/firc0001_review)
4+set -u
5+cd "$HOME/firc0001_review"
6+S="$1"; shift
7+NAME=$(basename "$S" | sed -E 's/\.(py|sh)$//')
8+LOG="logs/$NAME.log"; DONE="logs/$NAME.done"
9+rm -f "$DONE"
10+PY="$HOME/UQO_RESEARCH_DB/.venv/bin/python"
11+if [[ "$S" == *.sh ]]; then CMD="bash scripts/$S $*"; else CMD="$PY -I scripts/$S $*"; fi
12+nohup bash -c "echo START \$(date -Iseconds); nice -n 10 env OMP_NUM_THREADS=4 FIRC_THREADS=4 $CMD; echo RC=\$? END \$(date -Iseconds) > $DONE" > "$LOG" 2>&1 &
13+echo "started $NAME pid=$! log=$LOG"
added code/audit/spid/__init__.py +1 −0
@@ -0,0 +1 @@
1+"""spid package shim: hash-verified snapshot copies of the 2324b58 extractor modules (see spid_snapshot/SHA256SUMS)."""
added code/audit/spid/config.py +126 −0
@@ -0,0 +1,126 @@
1+# ============================================================
2+# SPID — configuration & paths
3+# ============================================================
4+from __future__ import annotations
5+
6+import logging
7+import os
8+from dataclasses import dataclass, field
9+from functools import lru_cache
10+from pathlib import Path
11+
12+# project-db/ (this file lives in project-db/spid/config.py)
13+PROJECT_ROOT = Path(__file__).resolve().parent.parent
14+
15+
16+def _load_dotenv() -> None:
17+ """Minimal .env loader (no python-dotenv dependency required)."""
18+ env_path = PROJECT_ROOT / ".env"
19+ if not env_path.exists():
20+ return
21+ for line in env_path.read_text().splitlines():
22+ line = line.strip()
23+ if not line or line.startswith("#") or "=" not in line:
24+ continue
25+ key, _, val = line.partition("=")
26+ key, val = key.strip(), val.strip().strip('"').strip("'")
27+ os.environ.setdefault(key, val)
28+
29+
30+_load_dotenv()
31+# The parent UQO_RESEARCH_DB that holds the parsed corpus.
32+RESEARCH_ROOT = PROJECT_ROOT.parent
33+SOURCE_DATA = RESEARCH_ROOT / "data"
34+
35+
36+def _parse_scope(spec: str) -> dict:
37+ out: dict = {}
38+ for part in (spec or "").split(";"):
39+ part = part.strip()
40+ if not part or ":" not in part:
41+ continue
42+ form, items = part.split(":", 1)
43+ out[form.strip()] = [i.strip() for i in items.split(",") if i.strip()]
44+ return out
45+
46+
47+@dataclass(frozen=True)
48+class Settings:
49+ """Resolved runtime configuration for the SPID platform."""
50+
51+ # --- source corpus (read-only) -------------------------------------
52+ source_data: Path = SOURCE_DATA
53+ parquet_dir: Path = SOURCE_DATA / "parquet"
54+ processed_dir: Path = SOURCE_DATA / "processed"
55+
56+ @property
57+ def filings_parquet(self) -> Path:
58+ return self.parquet_dir / "filings" / "filings.parquet"
59+
60+ @property
61+ def sections_parquet(self) -> Path:
62+ return self.parquet_dir / "filing_sections" / "filing_sections.parquet"
63+
64+ @property
65+ def companies_parquet(self) -> Path:
66+ return self.parquet_dir / "companies" / "companies.parquet"
67+
68+ # --- SPID outputs (writable) ---------------------------------------
69+ db_path: Path = PROJECT_ROOT / "data" / "db" / "spid.duckdb"
70+ out_dir: Path = PROJECT_ROOT / "data"
71+
72+ # --- extraction ----------------------------------------------------
73+ backend: str = field(
74+ default_factory=lambda: os.environ.get("SPID_BACKEND", "llm")
75+ ) # "llm" (gpt-4o-mini) | "rules"
76+ min_confidence: float = 0.45
77+ # which parsed sections feed the extractor, per form (None/empty = every section)
78+ section_scope: dict = field(default_factory=lambda: _parse_scope(os.environ.get(
79+ "SPID_SECTION_SCOPE",
80+ "10-K:business,risk_factors,properties,md_and_a,quantitative_disclosures")))
81+ max_section_chars: int = 600_000
82+
83+ # --- embeddings ----------------------------------------------------
84+ embedding_model: str = field(
85+ default_factory=lambda: os.environ.get("SPID_EMBED_MODEL", "all-MiniLM-L6-v2")
86+ )
87+
88+ # --- LLM backend (OpenAI-compatible) -------------------------------
89+ llm_base_url: str = field(
90+ default_factory=lambda: os.environ.get(
91+ "SPID_LLM_BASE_URL", "https://api.openai.com/v1"
92+ )
93+ )
94+ llm_model: str = field(
95+ default_factory=lambda: os.environ.get("SPID_LLM_MODEL", "gpt-4o-mini")
96+ )
97+ llm_api_key: str = field(
98+ default_factory=lambda: os.environ.get("OPENAI_API_KEY", "")
99+ or os.environ.get("MACLUSTR_API_KEY", "")
100+ )
101+ llm_timeout: float = 120.0
102+
103+ def ensure_dirs(self) -> None:
104+ self.db_path.parent.mkdir(parents=True, exist_ok=True)
105+ (self.out_dir / "exports").mkdir(parents=True, exist_ok=True)
106+
107+
108+@lru_cache(maxsize=1)
109+def get_settings() -> Settings:
110+ s = Settings()
111+ s.ensure_dirs()
112+ return s
113+
114+
115+def get_logger(name: str) -> logging.Logger:
116+ logger = logging.getLogger(f"spid.{name}")
117+ if not logger.handlers:
118+ h = logging.StreamHandler()
119+ h.setFormatter(
120+ logging.Formatter("%(asctime)s %(levelname)-7s %(name)s | %(message)s",
121+ datefmt="%H:%M:%S")
122+ )
123+ logger.addHandler(h)
124+ logger.setLevel(os.environ.get("SPID_LOG_LEVEL", "INFO"))
125+ logger.propagate = False
126+ return logger
added code/audit/spid/extract/__init__.py +0 −0
added code/audit/spid/extract/base.py +69 −0
@@ -0,0 +1,69 @@
1+# ============================================================
2+# SPID — extraction interface & record schema
3+# ============================================================
4+from __future__ import annotations
5+
6+import hashlib
7+from dataclasses import dataclass, field, asdict
8+from typing import Protocol
9+
10+
11+@dataclass
12+class ProjectMention:
13+ """A single project signal extracted from one filing section.
14+
15+ Mentions are the atomic unit; `resolve.py` merges many mentions
16+ (across filings & years) into a single Project with a timeline.
17+ """
18+ mention_id: str
19+ accession_number: str
20+ cik: str
21+ section_id: str
22+ section_name: str
23+
24+ project_name: str
25+ project_type: str
26+ description: str
27+ objective: str = ""
28+
29+ amount_usd: float | None = None
30+ amount_raw: str | None = None
31+ locations: list[str] = field(default_factory=list)
32+ technologies: list[str] = field(default_factory=list)
33+ partners: list[str] = field(default_factory=list)
34+ suppliers: list[str] = field(default_factory=list)
35+ benefits: list[str] = field(default_factory=list)
36+ risks: list[str] = field(default_factory=list)
37+ years: list[int] = field(default_factory=list)
38+ status: str = "mentioned"
39+
40+ confidence: float = 0.0
41+ backend: str = "rules"
42+
43+ def to_row(self) -> dict:
44+ d = asdict(self)
45+ # store list fields as JSON-friendly via duckdb list[str]
46+ return d
47+
48+ @staticmethod
49+ def make_id(accession: str, section_id: str, ptype: str, anchor: str) -> str:
50+ h = hashlib.sha1(f"{accession}|{section_id}|{ptype}|{anchor}".encode()).hexdigest()
51+ return h[:20]
52+
53+
54+class Extractor(Protocol):
55+ name: str
56+
57+ def extract(self, *, text: str, accession_number: str, cik: str,
58+ section_id: str, section_name: str) -> list[ProjectMention]:
59+ ...
60+
61+
62+def get_extractor(backend: str):
63+ """Factory. `backend` in {'rules','llm'}."""
64+ backend = (backend or "rules").lower()
65+ if backend == "llm":
66+ from spid.extract.llm import LLMExtractor
67+ return LLMExtractor()
68+ from spid.extract.rules import RuleExtractor
69+ return RuleExtractor()
added code/audit/spid/extract/llm.py +202 −0
@@ -0,0 +1,202 @@
1+# ============================================================
2+# SPID — LLM extractor (OpenAI-compatible; default gpt-4o-mini)
3+#
4+# Uses the chat-completions JSON mode to extract structured
5+# project records. Cost-controlled: only project-relevant
6+# windows of each section are sent to the model.
7+# ============================================================
8+from __future__ import annotations
9+
10+import json
11+import re
12+import time
13+
14+import httpx
15+
16+from spid import taxonomy as tx
17+from spid.config import get_logger, get_settings
18+from spid.extract.base import ProjectMention
19+from spid.extract.rules import _PROJECT_GATE
20+
21+log = get_logger("extract.llm")
22+
23+_SYSTEM = """You are a financial analyst extracting CORPORATE PROJECTS from SEC filing text.
24+A "project" is any concrete initiative, investment, program, expansion, construction,
25+product/R&D effort, acquisition/integration, technology deployment, partnership, or
26+strategic program the company is pursuing.
27+
28+Return STRICT JSON: {"projects": [ ... ]}. Each project object has:
29+ project_name : short descriptive name (use the official name if one is stated)
30+ project_type : EXACTLY ONE of the allowed types below
31+ description : 1 sentence, grounded in the text (no speculation)
32+ objective : the goal/why (or "")
33+ status : one of planned | in_progress | completed | mentioned
34+ amount_usd : numeric USD figure if stated (e.g. 1500000000), else null
35+ locations : array of places (city/state/country), else []
36+ technologies : array of technologies involved, else []
37+ partners : array of named partners/JV counterparties, else []
38+ benefits : array of expected benefits, else []
39+ risks : array of stated risks, else []
40+ years : array of 4-digit years mentioned, else []
41+ confidence : 0.0-1.0 (how clearly this is a real, concrete project)
42+
43+Allowed project_type values:
44+%s
45+
46+Rules:
47+- Extract ONLY real initiatives, not generic risk-factor boilerplate or hypotheticals.
48+- If the text describes no concrete project, return {"projects": []}.
49+- Do not invent facts. Numbers/locations must appear in the text.
50+- Prefer fewer high-quality projects over many vague ones.""" % "\n".join(
51+ f" - {k}: {v}" for k, v in tx.TYPE_LABELS.items()
52+)
53+
54+
55+def _focus(text: str, budget: int = 9000) -> str:
56+ """Keep only windows around project-signal hits to bound token cost."""
57+ if len(text) <= budget:
58+ return text
59+ spans: list[tuple[int, int]] = []
60+ for m in _PROJECT_GATE.finditer(text):
61+ spans.append((max(0, m.start() - 220), min(len(text), m.end() + 380)))
62+ if sum(b - a for a, b in spans) > budget * 1.4:
63+ break
64+ if not spans:
65+ return text[:budget]
66+ # merge overlapping windows
67+ spans.sort()
68+ merged = [spans[0]]
69+ for a, b in spans[1:]:
70+ if a <= merged[-1][1]:
71+ merged[-1] = (merged[-1][0], max(merged[-1][1], b))
72+ else:
73+ merged.append((a, b))
74+ chunks, total = [], 0
75+ for a, b in merged:
76+ chunks.append(text[a:b])
77+ total += b - a
78+ if total >= budget:
79+ break
80+ return "\n...\n".join(chunks)
81+
82+
83+class LLMExtractor:
84+ name = "llm"
85+
86+ def __init__(self) -> None:
87+ self.s = get_settings()
88+ if not self.s.llm_api_key:
89+ raise RuntimeError("No LLM API key configured (OPENAI_API_KEY / MACLUSTR_API_KEY).")
90+ self._client = httpx.Client(
91+ base_url=self.s.llm_base_url,
92+ headers={"Authorization": f"Bearer {self.s.llm_api_key}"},
93+ timeout=self.s.llm_timeout,
94+ )
95+
96+ def _call(self, content: str, retries: int = 3) -> dict:
97+ payload = {
98+ "model": self.s.llm_model,
99+ "messages": [
100+ {"role": "system", "content": _SYSTEM},
101+ {"role": "user", "content": content},
102+ ],
103+ "temperature": 0,
104+ "response_format": {"type": "json_object"},
105+ "max_tokens": 2000,
106+ }
107+ last = None
108+ for attempt in range(retries):
109+ try:
110+ r = self._client.post("/chat/completions", json=payload)
111+ if r.status_code == 429 or r.status_code >= 500:
112+ raise httpx.HTTPStatusError("retryable", request=r.request, response=r)
113+ r.raise_for_status()
114+ txt = r.json()["choices"][0]["message"]["content"]
115+ return json.loads(txt)
116+ except (httpx.HTTPError, json.JSONDecodeError, KeyError) as e:
117+ last = e
118+ time.sleep(1.5 * (attempt + 1))
119+ log.warning("LLM call failed after %d retries: %s", retries, last)
120+ return {"projects": []}
121+
122+ def extract(self, *, text: str, accession_number: str, cik: str,
123+ section_id: str, section_name: str) -> list[ProjectMention]:
124+ if not text or not _PROJECT_GATE.search(text):
125+ return []
126+ content = (f"SECTION: {section_name}\n\n" + _focus(text))[:14000]
127+ data = self._call(content)
128+ out: list[ProjectMention] = []
129+ for p in data.get("projects", []) or []:
130+ if not isinstance(p, dict):
131+ continue
132+ ptype = self._norm_type(p.get("project_type", ""))
133+ desc = str(p.get("description", "")).strip()[:600]
134+ if not desc:
135+ continue
136+ amount = p.get("amount_usd")
137+ try:
138+ amount = float(amount) if amount not in (None, "", "null") else None
139+ except (TypeError, ValueError):
140+ amount = None
141+ anchor = f"{ptype}:{str(p.get('project_name',''))[:40]}"
142+ conf = p.get("confidence", 0.6)
143+ try:
144+ conf = float(conf)
145+ except (TypeError, ValueError):
146+ conf = 0.6
147+ out.append(ProjectMention(
148+ mention_id=ProjectMention.make_id(accession_number, section_id, ptype, anchor),
149+ accession_number=accession_number, cik=cik,
150+ section_id=section_id, section_name=section_name,
151+ project_name=str(p.get("project_name") or tx.TYPE_LABELS.get(ptype, ptype))[:200],
152+ project_type=ptype, description=desc,
153+ objective=str(p.get("objective", ""))[:400],
154+ amount_usd=amount,
155+ amount_raw=(f"${amount:,.0f}" if amount else None),
156+ locations=_strlist(p.get("locations")),
157+ technologies=_strlist(p.get("technologies")) or tx.extract_technologies(desc),
158+ partners=_strlist(p.get("partners")),
159+ benefits=_strlist(p.get("benefits")),
160+ risks=_strlist(p.get("risks")),
161+ years=_intlist(p.get("years")) or tx.extract_years(desc),
162+ status=self._norm_status(p.get("status")),
163+ confidence=round(max(0.0, min(1.0, conf)), 3),
164+ backend=self.name,
165+ ))
166+ return out
167+
168+ @staticmethod
169+ def _norm_type(raw: str) -> str:
170+ raw = (raw or "").strip().lower().replace(" ", "_").replace("-", "_")
171+ if raw in tx.PROJECT_TYPES:
172+ return raw
173+ # fuzzy contains match against labels/keys
174+ for k, label in tx.TYPE_LABELS.items():
175+ if raw and (raw in k or raw in label.lower().replace(" ", "_")):
176+ return k
177+ return "technology_deployment"
178+
179+ @staticmethod
180+ def _norm_status(raw) -> str:
181+ raw = (str(raw or "")).strip().lower()
182+ return raw if raw in ("planned", "in_progress", "completed", "mentioned") else "mentioned"
183+
184+
185+def _strlist(v) -> list[str]:
186+ if not v:
187+ return []
188+ if isinstance(v, str):
189+ v = [v]
190+ return [str(x).strip()[:120] for x in v if str(x).strip()][:8]
191+
192+
193+def _intlist(v) -> list[int]:
194+ out = []
195+ for x in (v or []):
196+ try:
197+ y = int(str(x)[:4])
198+ if 1980 <= y <= 2040:
199+ out.append(y)
200+ except (TypeError, ValueError):
201+ continue
202+ return sorted(set(out))
added code/audit/spid/extract/rules.py +120 −0
@@ -0,0 +1,120 @@
1+# ============================================================
2+# SPID — deterministic rule-based extractor
3+#
4+# High-precision baseline that needs NO LLM. Scans section text
5+# sentence-by-sentence for project signals, classifies the type,
6+# and pulls money / location / tech / year / status.
7+# ============================================================
8+from __future__ import annotations
9+
10+import re
11+
12+from spid import taxonomy as tx
13+from spid.extract.base import ProjectMention
14+
15+# split into sentence-ish units; keep it cheap for 100k+ sections
16+_SENT_SPLIT = re.compile(r"(?<=[.;:!?])\s+(?=[A-Z(\"$])")
17+_WS = re.compile(r"\s+")
18+# generic "project-ish" gate so we skip boilerplate sentences fast
19+_PROJECT_GATE = re.compile(
20+ r"\b(project|initiative|program|invest|expan|construct|build|launch|develop|"
21+ r"deploy|implement|acqui|partner|facilit|capacity|capital|migrat|transform|"
22+ r"pipeline|plant|data ?cent|automat|sustainab|renewable|restructur)\w*",
23+ re.IGNORECASE,
24+)
25+_NAMED_PROJECT = re.compile(
26+ r"\b((?:Project|Program|Initiative)\s+[A-Z][A-Za-z0-9\-]+"
27+ r"|[A-Z][A-Za-z0-9\-]+\s+(?:Project|Program|Initiative))\b"
28+)
29+
30+
31+def _clean(s: str) -> str:
32+ return _WS.sub(" ", s).strip()
33+
34+
35+class RuleExtractor:
36+ name = "rules"
37+
38+ def extract(self, *, text: str, accession_number: str, cik: str,
39+ section_id: str, section_name: str) -> list[ProjectMention]:
40+ if not text:
41+ return []
42+ text = text[:600_000]
43+ out: list[ProjectMention] = []
44+ seen: set[str] = set()
45+
46+ for raw_sent in _SENT_SPLIT.split(text):
47+ sent = _clean(raw_sent)
48+ if not (20 <= len(sent) <= 700):
49+ continue
50+ if not _PROJECT_GATE.search(sent):
51+ continue
52+
53+ low = sent.lower()
54+ matched_types: list[str] = []
55+ for ptype, regexes in tx.TYPE_SIGNALS.items():
56+ if any(rx.search(low) for rx in regexes):
57+ matched_types.append(ptype)
58+ if not matched_types:
59+ continue
60+
61+ amount, amount_raw = tx.extract_money(sent)
62+ locations = tx.extract_locations(sent)
63+ techs = tx.extract_technologies(sent)
64+ years = tx.extract_years(sent)
65+ status = tx.detect_status(sent)
66+ nm = _NAMED_PROJECT.search(sent)
67+
68+ # keep at most the 2 strongest types for one sentence
69+ for ptype in matched_types[:2]:
70+ anchor = f"{ptype}:{(locations[0] if locations else '')}:{int(amount or 0)}"
71+ if anchor in seen:
72+ continue
73+ seen.add(anchor)
74+
75+ conf = self._confidence(ptype, amount, locations, years, status, nm, low)
76+ if conf < 0.30:
77+ continue
78+
79+ pname = nm.group(1) if nm else self._synth_name(ptype, locations, techs)
80+ out.append(ProjectMention(
81+ mention_id=ProjectMention.make_id(accession_number, section_id, ptype, anchor),
82+ accession_number=accession_number, cik=cik,
83+ section_id=section_id, section_name=section_name,
84+ project_name=pname, project_type=ptype,
85+ description=sent[:500],
86+ amount_usd=amount, amount_raw=amount_raw,
87+ locations=locations, technologies=techs, years=years,
88+ status=status, confidence=round(conf, 3), backend=self.name,
89+ ))
90+ return out
91+
92+ @staticmethod
93+ def _synth_name(ptype: str, locations: list[str], techs: list[str]) -> str:
94+ label = tx.TYPE_LABELS.get(ptype, ptype)
95+ if locations:
96+ return f"{label} — {locations[0]}"
97+ if techs:
98+ return f"{label} ({techs[0]})"
99+ return label
100+
101+ @staticmethod
102+ def _confidence(ptype, amount, locations, years, status, nm, low) -> float:
103+ c = 0.40
104+ if nm:
105+ c += 0.20
106+ if amount:
107+ c += 0.18
108+ if locations:
109+ c += 0.10
110+ if years:
111+ c += 0.06
112+ if status in ("planned", "in_progress", "completed"):
113+ c += 0.08
114+ # strong, specific phrasings
115+ if re.search(r"\b(announced|committed|invest\w+|will (build|open|invest))\b", low):
116+ c += 0.06
117+ # de-weight vague risk-factor / forward-looking hedges
118+ if re.search(r"\b(may|might|could|potential\w*|if we|uncertain)\b", low):
119+ c -= 0.08
120+ return max(0.0, min(1.0, c))
added code/audit/spid/taxonomy.py +261 −0
@@ -0,0 +1,261 @@
1+# ============================================================
2+# SPID — project taxonomy & lightweight value extractors
3+#
4+# Shared by both the rule-based and the LLM extractor so that
5+# `project_type`, `technology`, money, location and dates are
6+# normalised the same way regardless of backend.
7+# ============================================================
8+from __future__ import annotations
9+
10+import re
11+
12+# ----------------------------------------------------------------------
13+# Canonical project types. `signals` are high-precision regex fragments
14+# (already lower-cased) used by the rule extractor and as the controlled
15+# vocabulary the LLM is asked to classify into.
16+# ----------------------------------------------------------------------
17+PROJECT_TYPES: dict[str, dict] = {
18+ "manufacturing_expansion": {
19+ "label": "Manufacturing / Capacity Expansion",
20+ "signals": [
21+ r"(expand|increas\w+|add\w*|ramp\w*|scal\w+|new|build\w*|construct\w*|open\w*)"
22+ r"[^.]{0,60}(manufactur\w+|production|plant|factory|facilit\w+|capacity|assembly|fab\b)",
23+ ],
24+ },
25+ "plant_construction": {
26+ "label": "Plant / Site Construction",
27+ "signals": [r"(construct\w*|build\w*|broke ground|commission\w*|new (plant|site|facility))"
28+ r"[^.]{0,60}(plant|facility|site|campus|complex|refinery|mill)"],
29+ },
30+ "data_center": {
31+ "label": "Data Center",
32+ "signals": [r"data\s?cent(er|re)s?", r"hyperscal\w+", r"server farm"],
33+ },
34+ "cloud_migration": {
35+ "label": "Cloud Migration",
36+ "signals": [r"(migrat\w+|mov\w+|transition\w*)[^.]{0,40}(cloud)",
37+ r"cloud (migration|platform|adoption|infrastructure|first)"],
38+ },
39+ "ai_initiative": {
40+ "label": "AI / ML Initiative",
41+ "signals": [r"\b(artificial intelligence|machine learning|generative ai|gen ?ai|deep learning|"
42+ r"large language model|\bllm\b|neural network)\b",
43+ r"\bai[- ](powered|driven|enabled|capabilit\w+|platform|model|tools?)\b"],
44+ },
45+ "digital_transformation": {
46+ "label": "Digital Transformation",
47+ "signals": [r"digital (transformation|transition|initiative|strateg\w+|platform)",
48+ r"digitali[sz]ation"],
49+ },
50+ "erp_implementation": {
51+ "label": "ERP / Systems Implementation",
52+ "signals": [r"\b(erp|sap|oracle|workday|s/?4 ?hana)\b[^.]{0,40}(implement\w+|deploy\w+|migrat\w+|system|platform)",
53+ r"(implement\w+|deploy\w+|upgrad\w+)[^.]{0,40}\berp\b",
54+ r"enterprise resource planning"],
55+ },
56+ "rd_program": {
57+ "label": "R&D Program",
58+ "signals": [r"research and development", r"\br&d\b",
59+ r"(develop\w+|invest\w+)[^.]{0,40}(next[- ]generation|new technolog\w+|new product)"],
60+ },
61+ "product_launch": {
62+ "label": "New Product / Launch",
63+ "signals": [r"(launch\w*|introduc\w+|releas\w+|roll\w*[- ]?out|unveil\w+)"
64+ r"[^.]{0,50}(product|platform|service|offering|solution|model|line)",
65+ r"new product (line|launch|introduction)"],
66+ },
67+ "drug_pipeline": {
68+ "label": "Drug / Clinical Pipeline",
69+ "signals": [r"(phase\s?(i{1,3}|1|2|3)\b)", r"clinical trial", r"\bfda (approval|clearance|submission)\b",
70+ r"(drug|therap\w+|candidate|molecule|pipeline)[^.]{0,40}(develop\w+|trial|approval)"],
71+ },
72+ "ma_integration": {
73+ "label": "M&A / Integration",
74+ "signals": [r"(acqui\w+|merg\w+|takeover|business combination)",
75+ r"integration of [a-z]", r"purchase of [a-z]"],
76+ },
77+ "geographic_expansion": {
78+ "label": "Geographic Expansion",
79+ "signals": [r"(expand\w*|enter\w*|launch\w*|establish\w*)[^.]{0,50}"
80+ r"(market|region|countr\w+|geograph\w+|international|overseas|abroad)",
81+ r"market entry"],
82+ },
83+ "capex_program": {
84+ "label": "Capital Expenditure Program",
85+ "signals": [r"capital (expenditure|expenditures|investment|spending|program|project|budget)",
86+ r"\bcapex\b", r"invest\w+[^.]{0,30}(\$|usd)"],
87+ },
88+ "supply_chain": {
89+ "label": "Supply Chain Initiative",
90+ "signals": [r"supply chain", r"(reshor\w+|nearshor\w+|onshor\w+|offshor\w+)",
91+ r"(distribution|logistics|fulfillment|warehouse|inventory)[^.]{0,40}"
92+ r"(network|center|optimi[sz]\w+|expansion|investment)"],
93+ },
94+ "automation": {
95+ "label": "Automation / Robotics",
96+ "signals": [r"automat\w+", r"robot\w+", r"(industrial|warehouse) automation"],
97+ },
98+ "sustainability": {
99+ "label": "Sustainability / ESG",
100+ "signals": [r"sustainabilit\w+", r"\besg\b", r"net[- ]zero", r"carbon (neutral|reduction|footprint|emissions?)",
101+ r"decarboni[sz]\w+", r"emission reduction"],
102+ },
103+ "energy_transition": {
104+ "label": "Energy Transition / Renewables",
105+ "signals": [r"renewable (energy|power)", r"(solar|wind|hydrogen|geothermal|battery storage)[^.]{0,40}"
106+ r"(project|facility|farm|plant|capacity|investment)",
107+ r"clean energy", r"energy transition"],
108+ },
109+ "infrastructure": {
110+ "label": "Infrastructure Investment",
111+ "signals": [r"infrastructure (investment|project|upgrade|expansion|spending|build\w*)",
112+ r"(network|grid|pipeline|broadband|fiber) (expansion|build\w*|upgrade|deployment)"],
113+ },
114+ "cost_reduction": {
115+ "label": "Cost Reduction / Restructuring",
116+ "signals": [r"(cost (reduction|savings|cutting)|restructur\w+|efficiency (program|initiative)|"
117+ r"productivity (program|initiative)|streamlin\w+|optimi[sz]ation (program|initiative))"],
118+ },
119+ "partnership": {
120+ "label": "Strategic Partnership",
121+ "signals": [r"(strategic (partnership|alliance|collaboration)|joint venture|\bjv\b|"
122+ r"(partner\w+|collaborat\w+|team\w* up)[^.]{0,40}(with))"],
123+ },
124+ "technology_deployment": {
125+ "label": "Technology Deployment",
126+ "signals": [r"(deploy\w+|implement\w+|roll\w*[- ]?out|adopt\w+|install\w+)[^.]{0,50}"
127+ r"(technolog\w+|system|platform|software|solution|tool)"],
128+ },
129+}
130+
131+# ----------------------------------------------------------------------
132+# Technology vocabulary -> canonical tag
133+# ----------------------------------------------------------------------
134+TECHNOLOGY_TERMS: dict[str, str] = {
135+ r"\b(artificial intelligence|machine learning|generative ai|gen ?ai|deep learning|llm|neural)\b": "AI/ML",
136+ r"\bcloud\b": "Cloud",
137+ r"\b(erp|sap|oracle|workday|s/?4 ?hana|peoplesoft)\b": "ERP",
138+ r"\b(robot\w+|automation|cobots?)\b": "Robotics/Automation",
139+ r"\b(semiconductor|chip|wafer|fab|node\b|nanometer|foundry)\b": "Semiconductors",
140+ r"\b(solar|wind|renewable|hydrogen|geothermal)\b": "Renewable Energy",
141+ r"\b(battery|batteries|energy storage|lithium)\b": "Battery/Storage",
142+ r"\b(ev|electric vehicle|charging)\b": "Electric Vehicles",
143+ r"\b(blockchain|crypto|digital asset|tokeni[sz])\b": "Blockchain",
144+ r"\b(5g|broadband|fiber|fibre|network infrastructure)\b": "Connectivity",
145+ r"\b(data\s?cent(er|re)|hyperscale|server)\b": "Data Center",
146+ r"\b(cyber ?security|encryption|zero trust)\b": "Cybersecurity",
147+ r"\b(iot|internet of things|sensors?|edge computing)\b": "IoT/Edge",
148+ r"\b(quantum)\b": "Quantum",
149+ r"\b(biolog\w+|gene|mrna|crispr|cell therapy|molecul\w+)\b": "Biotech",
150+ r"\b(saas|software[- ]as[- ]a[- ]service|platform)\b": "Software/SaaS",
151+}
152+
153+# ----------------------------------------------------------------------
154+# Status keywords -> canonical status
155+# ----------------------------------------------------------------------
156+STATUS_TERMS: list[tuple[str, str]] = [
157+ (r"\b(completed|commission\w+|operational|in service|came online|opened|finalized)\b", "completed"),
158+ (r"\b(under construction|underway|in progress|ongoing|ramping|being (built|implemented))\b", "in_progress"),
159+ (r"\b(announc\w+|plan\w+|intend\w+|expect\w+|will (build|invest|launch|open)|propos\w+|"
160+ r"anticipat\w+|target\w+|by (20\d\d)|slated)\b", "planned"),
161+]
162+
163+# ----------------------------------------------------------------------
164+# Geography
165+# ----------------------------------------------------------------------
166+US_STATES = {
167+ "alabama","alaska","arizona","arkansas","california","colorado","connecticut","delaware",
168+ "florida","georgia","hawaii","idaho","illinois","indiana","iowa","kansas","kentucky",
169+ "louisiana","maine","maryland","massachusetts","michigan","minnesota","mississippi",
170+ "missouri","montana","nebraska","nevada","new hampshire","new jersey","new mexico",
171+ "new york","north carolina","north dakota","ohio","oklahoma","oregon","pennsylvania",
172+ "rhode island","south carolina","south dakota","tennessee","texas","utah","vermont",
173+ "virginia","washington","west virginia","wisconsin","wyoming",
174+}
175+COUNTRIES = {
176+ "united states","u.s.","usa","canada","mexico","china","india","japan","germany","france",
177+ "united kingdom","u.k.","ireland","brazil","singapore","south korea","korea","taiwan",
178+ "vietnam","thailand","malaysia","indonesia","australia","spain","italy","netherlands",
179+ "switzerland","sweden","poland","israel","saudi arabia","united arab emirates","uae",
180+ "philippines","czech republic","hungary","turkey","egypt","south africa","argentina","chile",
181+}
182+
183+_MONEY_RE = re.compile(
184+ r"(?:US)?\$\s?(\d[\d,]*\.?\d*)\s?(billion|bn|million|mm|mn|m\b|thousand|k\b|trillion)?",
185+ re.IGNORECASE,
186+)
187+_MULT = {
188+ "trillion": 1e12, "billion": 1e9, "bn": 1e9, "million": 1e6, "mm": 1e6, "mn": 1e6,
189+ "m": 1e6, "thousand": 1e3, "k": 1e3,
190+}
191+_YEAR_RE = re.compile(r"\b(?:fiscal\s+|fy\s?)?(20[0-3]\d|19[89]\d)\b", re.IGNORECASE)
192+_QUARTER_RE = re.compile(r"\bQ([1-4])\s?(20\d\d)\b", re.IGNORECASE)
193+_TECH_COMPILED = [(re.compile(p, re.IGNORECASE), tag) for p, tag in TECHNOLOGY_TERMS.items()]
194+_STATUS_COMPILED = [(re.compile(p, re.IGNORECASE), s) for p, s in STATUS_TERMS]
195+_CITY_STATE_RE = re.compile(r"\bin\s+([A-Z][a-zA-Z.]+(?:\s[A-Z][a-zA-Z.]+){0,2}),\s+([A-Z][a-zA-Z]+)")
196+
197+
198+def extract_money(text: str) -> tuple[float | None, str | None]:
199+ """Return (usd_amount, raw_span) for the largest explicit dollar figure."""
200+ best_val, best_raw = None, None
201+ for m in _MONEY_RE.finditer(text):
202+ try:
203+ num = float(m.group(1).replace(",", ""))
204+ except ValueError:
205+ continue
206+ unit = (m.group(2) or "").lower().strip()
207+ val = num * _MULT.get(unit, 1.0)
208+ if best_val is None or val > best_val:
209+ best_val, best_raw = val, m.group(0).strip()
210+ return best_val, best_raw
211+
212+
213+def extract_locations(text: str) -> list[str]:
214+ low = text.lower()
215+ found: list[str] = []
216+ for st in US_STATES:
217+ if re.search(rf"\b{re.escape(st)}\b", low):
218+ found.append(st.title())
219+ for co in COUNTRIES:
220+ if re.search(rf"\b{re.escape(co)}\b", low):
221+ found.append(co.upper() if len(co) <= 4 else co.title())
222+ for m in _CITY_STATE_RE.finditer(text):
223+ found.append(f"{m.group(1)}, {m.group(2)}")
224+ # de-dup preserving order
225+ seen, out = set(), []
226+ for f in found:
227+ k = f.lower()
228+ if k not in seen:
229+ seen.add(k); out.append(f)
230+ return out[:6]
231+
232+
233+def extract_years(text: str) -> list[int]:
234+ yrs = {int(y) for y in _YEAR_RE.findall(text)}
235+ for m in _QUARTER_RE.finditer(text):
236+ yrs.add(int(m.group(2)))
237+ return sorted(yrs)
238+
239+
240+def extract_technologies(text: str) -> list[str]:
241+ tags = []
242+ for rx, tag in _TECH_COMPILED:
243+ if rx.search(text) and tag not in tags:
244+ tags.append(tag)
245+ return tags
246+
247+
248+def detect_status(text: str) -> str:
249+ for rx, status in _STATUS_COMPILED:
250+ if rx.search(text):
251+ return status
252+ return "mentioned"
253+
254+
255+# pre-compile project-type signals
256+TYPE_SIGNALS = {
257+ ptype: [re.compile(sig, re.IGNORECASE) for sig in spec["signals"]]
258+ for ptype, spec in PROJECT_TYPES.items()
259+}
260+TYPE_LABELS = {k: v["label"] for k, v in PROJECT_TYPES.items()}
261+ALL_TYPES = list(PROJECT_TYPES.keys())
added code/audit/spid_snapshot/SHA256SUMS +9 −0
@@ -0,0 +1,9 @@
1+4ba1811f9513ab472855764ba8d09363060fed6e7d009f68f5b2749e85833ab1 worktrees/FIRC-0001/code/audit/spid_snapshot/base.py
2+8c4a273631e0015867a1c88d92e284faca906ea16e9af2595dfa5cb06d5b8d06 worktrees/FIRC-0001/code/audit/spid_snapshot/config.py
3+c53536becfe273f03882c309ba6ac79d4423c5bf1320865d044a0d00effada90 worktrees/FIRC-0001/code/audit/spid_snapshot/ingest.py
4+ae7f3f61a8de36842ba8c99a566883465fe8d6672ec723ee438f963e50d4eb89 worktrees/FIRC-0001/code/audit/spid_snapshot/llm.py
5+6135f39adb9e78b7e50b81a9034db07e45f30fcb3a329df5c6c9a79bad028712 worktrees/FIRC-0001/code/audit/spid_snapshot/pipeline.py
6+925ab1258b2a364f1c9ad5be5817b4d815fe11e7fca5ad32f3ac71b5be59652d worktrees/FIRC-0001/code/audit/spid_snapshot/resolve.py
7+7f191c18b0426d83931fa6f1f461f3a2d43bcc1d151a14a6e084f24db6766807 worktrees/FIRC-0001/code/audit/spid_snapshot/rules.py
8+1219c923fff72b9946bc068baacd70802fd5fab86766e62141dbdd2ea066a54f worktrees/FIRC-0001/code/audit/spid_snapshot/store.py
9+92ad69bcc4e448d03496943221fdcb78b4326baa7db682a6b6713c260c9c4e7a worktrees/FIRC-0001/code/audit/spid_snapshot/taxonomy.py
added code/audit/spid_snapshot/base.py +69 −0
@@ -0,0 +1,69 @@
1+# ============================================================
2+# SPID — extraction interface & record schema
3+# ============================================================
4+from __future__ import annotations
5+
6+import hashlib
7+from dataclasses import dataclass, field, asdict
8+from typing import Protocol
9+
10+
11+@dataclass
12+class ProjectMention:
13+ """A single project signal extracted from one filing section.
14+
15+ Mentions are the atomic unit; `resolve.py` merges many mentions
16+ (across filings & years) into a single Project with a timeline.
17+ """
18+ mention_id: str
19+ accession_number: str
20+ cik: str
21+ section_id: str
22+ section_name: str
23+
24+ project_name: str
25+ project_type: str
26+ description: str
27+ objective: str = ""
28+
29+ amount_usd: float | None = None
30+ amount_raw: str | None = None
31+ locations: list[str] = field(default_factory=list)
32+ technologies: list[str] = field(default_factory=list)
33+ partners: list[str] = field(default_factory=list)
34+ suppliers: list[str] = field(default_factory=list)
35+ benefits: list[str] = field(default_factory=list)
36+ risks: list[str] = field(default_factory=list)
37+ years: list[int] = field(default_factory=list)
38+ status: str = "mentioned"
39+
40+ confidence: float = 0.0
41+ backend: str = "rules"
42+
43+ def to_row(self) -> dict:
44+ d = asdict(self)
45+ # store list fields as JSON-friendly via duckdb list[str]
46+ return d
47+
48+ @staticmethod
49+ def make_id(accession: str, section_id: str, ptype: str, anchor: str) -> str:
50+ h = hashlib.sha1(f"{accession}|{section_id}|{ptype}|{anchor}".encode()).hexdigest()
51+ return h[:20]
52+
53+
54+class Extractor(Protocol):
55+ name: str
56+
57+ def extract(self, *, text: str, accession_number: str, cik: str,
58+ section_id: str, section_name: str) -> list[ProjectMention]:
59+ ...
60+
61+
62+def get_extractor(backend: str):
63+ """Factory. `backend` in {'rules','llm'}."""
64+ backend = (backend or "rules").lower()
65+ if backend == "llm":
66+ from spid.extract.llm import LLMExtractor
67+ return LLMExtractor()
68+ from spid.extract.rules import RuleExtractor
69+ return RuleExtractor()
added code/audit/spid_snapshot/config.py +126 −0
@@ -0,0 +1,126 @@
1+# ============================================================
2+# SPID — configuration & paths
3+# ============================================================
4+from __future__ import annotations
5+
6+import logging
7+import os
8+from dataclasses import dataclass, field
9+from functools import lru_cache
10+from pathlib import Path
11+
12+# project-db/ (this file lives in project-db/spid/config.py)
13+PROJECT_ROOT = Path(__file__).resolve().parent.parent
14+
15+
16+def _load_dotenv() -> None:
17+ """Minimal .env loader (no python-dotenv dependency required)."""
18+ env_path = PROJECT_ROOT / ".env"
19+ if not env_path.exists():
20+ return
21+ for line in env_path.read_text().splitlines():
22+ line = line.strip()
23+ if not line or line.startswith("#") or "=" not in line:
24+ continue
25+ key, _, val = line.partition("=")
26+ key, val = key.strip(), val.strip().strip('"').strip("'")
27+ os.environ.setdefault(key, val)
28+
29+
30+_load_dotenv()
31+# The parent UQO_RESEARCH_DB that holds the parsed corpus.
32+RESEARCH_ROOT = PROJECT_ROOT.parent
33+SOURCE_DATA = RESEARCH_ROOT / "data"
34+
35+
36+def _parse_scope(spec: str) -> dict:
37+ out: dict = {}
38+ for part in (spec or "").split(";"):
39+ part = part.strip()
40+ if not part or ":" not in part:
41+ continue
42+ form, items = part.split(":", 1)
43+ out[form.strip()] = [i.strip() for i in items.split(",") if i.strip()]
44+ return out
45+
46+
47+@dataclass(frozen=True)
48+class Settings:
49+ """Resolved runtime configuration for the SPID platform."""
50+
51+ # --- source corpus (read-only) -------------------------------------
52+ source_data: Path = SOURCE_DATA
53+ parquet_dir: Path = SOURCE_DATA / "parquet"
54+ processed_dir: Path = SOURCE_DATA / "processed"
55+
56+ @property
57+ def filings_parquet(self) -> Path:
58+ return self.parquet_dir / "filings" / "filings.parquet"
59+
60+ @property
61+ def sections_parquet(self) -> Path:
62+ return self.parquet_dir / "filing_sections" / "filing_sections.parquet"
63+
64+ @property
65+ def companies_parquet(self) -> Path:
66+ return self.parquet_dir / "companies" / "companies.parquet"
67+
68+ # --- SPID outputs (writable) ---------------------------------------
69+ db_path: Path = PROJECT_ROOT / "data" / "db" / "spid.duckdb"
70+ out_dir: Path = PROJECT_ROOT / "data"
71+
72+ # --- extraction ----------------------------------------------------
73+ backend: str = field(
74+ default_factory=lambda: os.environ.get("SPID_BACKEND", "llm")
75+ ) # "llm" (gpt-4o-mini) | "rules"
76+ min_confidence: float = 0.45
77+ # which parsed sections feed the extractor, per form (None/empty = every section)
78+ section_scope: dict = field(default_factory=lambda: _parse_scope(os.environ.get(
79+ "SPID_SECTION_SCOPE",
80+ "10-K:business,risk_factors,properties,md_and_a,quantitative_disclosures")))
81+ max_section_chars: int = 600_000
82+
83+ # --- embeddings ----------------------------------------------------
84+ embedding_model: str = field(
85+ default_factory=lambda: os.environ.get("SPID_EMBED_MODEL", "all-MiniLM-L6-v2")
86+ )
87+
88+ # --- LLM backend (OpenAI-compatible) -------------------------------
89+ llm_base_url: str = field(
90+ default_factory=lambda: os.environ.get(
91+ "SPID_LLM_BASE_URL", "https://api.openai.com/v1"
92+ )
93+ )
94+ llm_model: str = field(
95+ default_factory=lambda: os.environ.get("SPID_LLM_MODEL", "gpt-4o-mini")
96+ )
97+ llm_api_key: str = field(
98+ default_factory=lambda: os.environ.get("OPENAI_API_KEY", "")
99+ or os.environ.get("MACLUSTR_API_KEY", "")
100+ )
101+ llm_timeout: float = 120.0
102+
103+ def ensure_dirs(self) -> None:
104+ self.db_path.parent.mkdir(parents=True, exist_ok=True)
105+ (self.out_dir / "exports").mkdir(parents=True, exist_ok=True)
106+
107+
108+@lru_cache(maxsize=1)
109+def get_settings() -> Settings:
110+ s = Settings()
111+ s.ensure_dirs()
112+ return s
113+
114+
115+def get_logger(name: str) -> logging.Logger:
116+ logger = logging.getLogger(f"spid.{name}")
117+ if not logger.handlers:
118+ h = logging.StreamHandler()
119+ h.setFormatter(
120+ logging.Formatter("%(asctime)s %(levelname)-7s %(name)s | %(message)s",
121+ datefmt="%H:%M:%S")
122+ )
123+ logger.addHandler(h)
124+ logger.setLevel(os.environ.get("SPID_LOG_LEVEL", "INFO"))
125+ logger.propagate = False
126+ return logger
added code/audit/spid_snapshot/ingest.py +135 −0
@@ -0,0 +1,135 @@
1+# ============================================================
2+# SPID — raw 10-K ingester
3+#
4+# The pre-parsed corpus only sectioned 10-Q & 8-K filings; the
5+# 10-K HTML was left raw. This module cleans the raw 10-K HTML
6+# and extracts the project-dense Items (1 Business, 1A Risk,
7+# 7 MD&A, 2 Properties) into a SPID-owned `spid_sections` table
8+# that `v_sections` unions in — so 10-Ks flow through the same
9+# extraction pipeline.
10+# ============================================================
11+from __future__ import annotations
12+
13+import glob
14+import re
15+import uuid
16+from pathlib import Path
17+
18+from spid.config import get_logger, get_settings
19+from spid.store import Store
20+
21+log = get_logger("ingest")
22+
23+RAW_SEC = get_settings().source_data / "raw" / "sec"
24+
25+# 10-K items we keep (where projects/capex/strategy concentrate)
26+_KEEP_ITEMS = {
27+ "1": "business", "1A": "risk_factors", "2": "properties",
28+ "7": "md_and_a", "7A": "quantitative_disclosures",
29+}
30+_ITEM_RE = re.compile(r"(?:^|\n)\s*ITEM\s+(\d+[AB]?)\s*[.\-:—–]", re.IGNORECASE)
31+_TAG_RE = re.compile(r"<[^>]+>")
32+_WS_RE = re.compile(r"[ \t]+")
33+_NL_RE = re.compile(r"\n{3,}")
34+
35+
36+def _clean_html(path: Path) -> str:
37+ raw = path.read_text(errors="ignore")
38+ try:
39+ from bs4 import BeautifulSoup
40+ soup = BeautifulSoup(raw, "lxml")
41+ for t in soup(["script", "style", "table"]):
42+ t.decompose()
43+ text = soup.get_text("\n")
44+ except Exception: # noqa: BLE001
45+ text = _TAG_RE.sub(" ", raw)
46+ import html as _html
47+ text = _html.unescape(text)
48+ text = _WS_RE.sub(" ", text)
49+ text = _NL_RE.sub("\n\n", text)
50+ return text
51+
52+
53+def _extract_items(text: str) -> list[tuple[str, str]]:
54+ matches = list(_ITEM_RE.finditer(text))
55+ if not matches:
56+ return [("full_body", text)]
57+ out: list[tuple[str, str]] = []
58+ for i, m in enumerate(matches):
59+ item = m.group(1).upper()
60+ if item not in _KEEP_ITEMS:
61+ continue
62+ start = m.start()
63+ end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
64+ body = text[start:end].strip()
65+ if len(body) > 400: # skip the table-of-contents stubs
66+ out.append((_KEEP_ITEMS[item], body[:600_000]))
67+ # keep the longest instance of each section (TOC vs real body)
68+ best: dict[str, str] = {}
69+ for name, body in out:
70+ if name not in best or len(body) > len(best[name]):
71+ best[name] = body
72+ return list(best.items())
73+
74+
75+def _find_primary(acc: str, cik: str) -> Path | None:
76+ d = RAW_SEC / cik / "10-K" / acc.replace("-", "")
77+ if not d.is_dir():
78+ return None
79+ cands = glob.glob(str(d / "*.htm")) + glob.glob(str(d / "*.html")) + glob.glob(str(d / "*.txt"))
80+ cands = [c for c in cands if "R" + "" not in Path(c).name or True]
81+ if not cands:
82+ return None
83+ return Path(max(cands, key=lambda p: Path(p).stat().st_size))
84+
85+
86+_SPID_SECTIONS_DDL = """
87+CREATE TABLE IF NOT EXISTS spid_sections (
88+ section_id VARCHAR PRIMARY KEY,
89+ accession_number VARCHAR,
90+ section_name VARCHAR,
91+ item_number VARCHAR,
92+ text VARCHAR,
93+ word_count INTEGER
94+);
95+"""
96+
97+
98+def ingest_10k(store: Store, *, ciks: list[str] | None = None, limit: int | None = None) -> dict:
99+ con = store.con
100+ con.execute(_SPID_SECTIONS_DDL)
101+ where = "form_type='10-K'"
102+ if ciks:
103+ q = ",".join("'" + c + "'" for c in ciks)
104+ where += f" AND cik IN ({q})"
105+ sql = f"SELECT accession_number, cik FROM src_filings WHERE {where} ORDER BY filing_date DESC"
106+ if limit:
107+ sql += f" LIMIT {int(limit)}"
108+ filings = con.execute(sql).fetchall()
109+
110+ rows, done, ok = [], 0, 0
111+ for acc, cik in filings:
112+ done += 1
113+ # skip if already ingested
114+ if con.execute("SELECT 1 FROM spid_sections WHERE accession_number=? LIMIT 1", [acc]).fetchone():
115+ ok += 1
116+ continue
117+ path = _find_primary(acc, cik)
118+ if not path:
119+ continue
120+ try:
121+ text = _clean_html(path)
122+ for name, body in _extract_items(text):
123+ rows.append((str(uuid.uuid4()), acc, name, "", body, len(body.split())))
124+ ok += 1
125+ except Exception as e: # noqa: BLE001
126+ log.warning("ingest failed %s: %s", acc, e)
127+ if len(rows) >= 400:
128+ con.executemany("INSERT OR REPLACE INTO spid_sections VALUES (?,?,?,?,?,?)", rows); rows = []
129+ if done % 50 == 0:
130+ log.info(" ... ingested %d/%d 10-Ks", done, len(filings))
131+ if rows:
132+ con.executemany("INSERT OR REPLACE INTO spid_sections VALUES (?,?,?,?,?,?)", rows)
133+ n = store.count("spid_sections")
134+ log.info("Ingested 10-K sections: %d filings processed, %d section rows total.", ok, n)
135+ return {"filings_processed": ok, "section_rows": n}
added code/audit/spid_snapshot/llm.py +202 −0
@@ -0,0 +1,202 @@
1+# ============================================================
2+# SPID — LLM extractor (OpenAI-compatible; default gpt-4o-mini)
3+#
4+# Uses the chat-completions JSON mode to extract structured
5+# project records. Cost-controlled: only project-relevant
6+# windows of each section are sent to the model.
7+# ============================================================
8+from __future__ import annotations
9+
10+import json
11+import re
12+import time
13+
14+import httpx
15+
16+from spid import taxonomy as tx
17+from spid.config import get_logger, get_settings
18+from spid.extract.base import ProjectMention
19+from spid.extract.rules import _PROJECT_GATE
20+
21+log = get_logger("extract.llm")
22+
23+_SYSTEM = """You are a financial analyst extracting CORPORATE PROJECTS from SEC filing text.
24+A "project" is any concrete initiative, investment, program, expansion, construction,
25+product/R&D effort, acquisition/integration, technology deployment, partnership, or
26+strategic program the company is pursuing.
27+
28+Return STRICT JSON: {"projects": [ ... ]}. Each project object has:
29+ project_name : short descriptive name (use the official name if one is stated)
30+ project_type : EXACTLY ONE of the allowed types below
31+ description : 1 sentence, grounded in the text (no speculation)
32+ objective : the goal/why (or "")
33+ status : one of planned | in_progress | completed | mentioned
34+ amount_usd : numeric USD figure if stated (e.g. 1500000000), else null
35+ locations : array of places (city/state/country), else []
36+ technologies : array of technologies involved, else []
37+ partners : array of named partners/JV counterparties, else []
38+ benefits : array of expected benefits, else []
39+ risks : array of stated risks, else []
40+ years : array of 4-digit years mentioned, else []
41+ confidence : 0.0-1.0 (how clearly this is a real, concrete project)
42+
43+Allowed project_type values:
44+%s
45+
46+Rules:
47+- Extract ONLY real initiatives, not generic risk-factor boilerplate or hypotheticals.
48+- If the text describes no concrete project, return {"projects": []}.
49+- Do not invent facts. Numbers/locations must appear in the text.
50+- Prefer fewer high-quality projects over many vague ones.""" % "\n".join(
51+ f" - {k}: {v}" for k, v in tx.TYPE_LABELS.items()
52+)
53+
54+
55+def _focus(text: str, budget: int = 9000) -> str:
56+ """Keep only windows around project-signal hits to bound token cost."""
57+ if len(text) <= budget:
58+ return text
59+ spans: list[tuple[int, int]] = []
60+ for m in _PROJECT_GATE.finditer(text):
61+ spans.append((max(0, m.start() - 220), min(len(text), m.end() + 380)))
62+ if sum(b - a for a, b in spans) > budget * 1.4:
63+ break
64+ if not spans:
65+ return text[:budget]
66+ # merge overlapping windows
67+ spans.sort()
68+ merged = [spans[0]]
69+ for a, b in spans[1:]:
70+ if a <= merged[-1][1]:
71+ merged[-1] = (merged[-1][0], max(merged[-1][1], b))
72+ else:
73+ merged.append((a, b))
74+ chunks, total = [], 0
75+ for a, b in merged:
76+ chunks.append(text[a:b])
77+ total += b - a
78+ if total >= budget:
79+ break
80+ return "\n...\n".join(chunks)
81+
82+
83+class LLMExtractor:
84+ name = "llm"
85+
86+ def __init__(self) -> None:
87+ self.s = get_settings()
88+ if not self.s.llm_api_key:
89+ raise RuntimeError("No LLM API key configured (OPENAI_API_KEY / MACLUSTR_API_KEY).")
90+ self._client = httpx.Client(
91+ base_url=self.s.llm_base_url,
92+ headers={"Authorization": f"Bearer {self.s.llm_api_key}"},
93+ timeout=self.s.llm_timeout,
94+ )
95+
96+ def _call(self, content: str, retries: int = 3) -> dict:
97+ payload = {
98+ "model": self.s.llm_model,
99+ "messages": [
100+ {"role": "system", "content": _SYSTEM},
101+ {"role": "user", "content": content},
102+ ],
103+ "temperature": 0,
104+ "response_format": {"type": "json_object"},
105+ "max_tokens": 2000,
106+ }
107+ last = None
108+ for attempt in range(retries):
109+ try:
110+ r = self._client.post("/chat/completions", json=payload)
111+ if r.status_code == 429 or r.status_code >= 500:
112+ raise httpx.HTTPStatusError("retryable", request=r.request, response=r)
113+ r.raise_for_status()
114+ txt = r.json()["choices"][0]["message"]["content"]
115+ return json.loads(txt)
116+ except (httpx.HTTPError, json.JSONDecodeError, KeyError) as e:
117+ last = e
118+ time.sleep(1.5 * (attempt + 1))
119+ log.warning("LLM call failed after %d retries: %s", retries, last)
120+ return {"projects": []}
121+
122+ def extract(self, *, text: str, accession_number: str, cik: str,
123+ section_id: str, section_name: str) -> list[ProjectMention]:
124+ if not text or not _PROJECT_GATE.search(text):
125+ return []
126+ content = (f"SECTION: {section_name}\n\n" + _focus(text))[:14000]
127+ data = self._call(content)
128+ out: list[ProjectMention] = []
129+ for p in data.get("projects", []) or []:
130+ if not isinstance(p, dict):
131+ continue
132+ ptype = self._norm_type(p.get("project_type", ""))
133+ desc = str(p.get("description", "")).strip()[:600]
134+ if not desc:
135+ continue
136+ amount = p.get("amount_usd")
137+ try:
138+ amount = float(amount) if amount not in (None, "", "null") else None
139+ except (TypeError, ValueError):
140+ amount = None
141+ anchor = f"{ptype}:{str(p.get('project_name',''))[:40]}"
142+ conf = p.get("confidence", 0.6)
143+ try:
144+ conf = float(conf)
145+ except (TypeError, ValueError):
146+ conf = 0.6
147+ out.append(ProjectMention(
148+ mention_id=ProjectMention.make_id(accession_number, section_id, ptype, anchor),
149+ accession_number=accession_number, cik=cik,
150+ section_id=section_id, section_name=section_name,
151+ project_name=str(p.get("project_name") or tx.TYPE_LABELS.get(ptype, ptype))[:200],
152+ project_type=ptype, description=desc,
153+ objective=str(p.get("objective", ""))[:400],
154+ amount_usd=amount,
155+ amount_raw=(f"${amount:,.0f}" if amount else None),
156+ locations=_strlist(p.get("locations")),
157+ technologies=_strlist(p.get("technologies")) or tx.extract_technologies(desc),
158+ partners=_strlist(p.get("partners")),
159+ benefits=_strlist(p.get("benefits")),
160+ risks=_strlist(p.get("risks")),
161+ years=_intlist(p.get("years")) or tx.extract_years(desc),
162+ status=self._norm_status(p.get("status")),
163+ confidence=round(max(0.0, min(1.0, conf)), 3),
164+ backend=self.name,
165+ ))
166+ return out
167+
168+ @staticmethod
169+ def _norm_type(raw: str) -> str:
170+ raw = (raw or "").strip().lower().replace(" ", "_").replace("-", "_")
171+ if raw in tx.PROJECT_TYPES:
172+ return raw
173+ # fuzzy contains match against labels/keys
174+ for k, label in tx.TYPE_LABELS.items():
175+ if raw and (raw in k or raw in label.lower().replace(" ", "_")):
176+ return k
177+ return "technology_deployment"
178+
179+ @staticmethod
180+ def _norm_status(raw) -> str:
181+ raw = (str(raw or "")).strip().lower()
182+ return raw if raw in ("planned", "in_progress", "completed", "mentioned") else "mentioned"
183+
184+
185+def _strlist(v) -> list[str]:
186+ if not v:
187+ return []
188+ if isinstance(v, str):
189+ v = [v]
190+ return [str(x).strip()[:120] for x in v if str(x).strip()][:8]
191+
192+
193+def _intlist(v) -> list[int]:
194+ out = []
195+ for x in (v or []):
196+ try:
197+ y = int(str(x)[:4])
198+ if 1980 <= y <= 2040:
199+ out.append(y)
200+ except (TypeError, ValueError):
201+ continue
202+ return sorted(set(out))
added code/audit/spid_snapshot/pipeline.py +95 −0
@@ -0,0 +1,95 @@
1+# ============================================================
2+# SPID — extraction engine
3+#
4+# Iterates enriched sections, runs the chosen extractor
5+# (rules or gpt-4o-mini), filters by confidence, and loads
6+# mentions into DuckDB. LLM calls are threaded (IO-bound).
7+# ============================================================
8+from __future__ import annotations
9+
10+from concurrent.futures import ThreadPoolExecutor, as_completed
11+
12+from spid.config import get_logger, get_settings
13+from spid.extract.base import get_extractor
14+from spid.store import Store
15+
16+log = get_logger("pipeline")
17+
18+
19+def _ctx_from_rows(rows: list[dict]) -> dict:
20+ ctx = {}
21+ for r in rows:
22+ ctx.setdefault(r["accession_number"], {
23+ "ticker": r.get("ticker"),
24+ "company_name": r.get("company_name") or r.get("filing_company"),
25+ "filing_company": r.get("filing_company"),
26+ "form_type": r.get("form_type"),
27+ "filing_date": r.get("filing_date"),
28+ "fiscal_year": r.get("fiscal_year"),
29+ "sector": r.get("sector"),
30+ })
31+ return ctx
32+
33+
34+def run_extraction(store: Store, *, backend: str | None = None,
35+ ciks: list[str] | None = None, forms: list[str] | None = None,
36+ limit_sections: int | None = None, min_words: int = 30,
37+ workers: int = 8, min_confidence: float | None = None,
38+ flush_every: int = 500) -> dict:
39+ s = get_settings()
40+ backend = backend or s.backend
41+ min_conf = s.min_confidence if min_confidence is None else min_confidence
42+ extractor = get_extractor(backend)
43+
44+ rows = store.sections_for(ciks=ciks, forms=forms, limit=limit_sections, min_words=min_words)
45+ ctx = _ctx_from_rows(rows)
46+ log.info("Extracting from %d sections | backend=%s | workers=%d", len(rows), backend, workers)
47+
48+ kept = 0
49+ buf = []
50+ done_rows: list[tuple] = []
51+
52+ def work(r: dict):
53+ return extractor.extract(text=r["text"], accession_number=r["accession_number"],
54+ cik=r["cik"], section_id=r["section_id"],
55+ section_name=r["section_name"])
56+
57+ def flush():
58+ nonlocal kept, buf, done_rows
59+ if buf:
60+ kept += store.upsert_mentions(buf, ctx)
61+ buf = []
62+ if done_rows:
63+ store.mark_extracted(done_rows)
64+ done_rows = []
65+
66+ done = 0
67+ if backend == "llm" and workers > 1:
68+ with ThreadPoolExecutor(max_workers=workers) as ex:
69+ futs = {ex.submit(work, r): r for r in rows}
70+ for fut in as_completed(futs):
71+ done += 1
72+ r = futs[fut]
73+ try:
74+ ms = [m for m in fut.result() if m.confidence >= min_conf]
75+ done_rows.append((r["section_id"], r["accession_number"], backend, len(ms)))
76+ except Exception as e: # noqa: BLE001
77+ log.warning("section failed: %s", e); ms = []
78+ buf.extend(ms)
79+ if len(buf) >= flush_every:
80+ flush()
81+ if done % 200 == 0:
82+ log.info(" ... %d/%d sections, %d mentions kept", done, len(rows), kept + len(buf))
83+ else:
84+ for r in rows:
85+ done += 1
86+ ms = [m for m in work(r) if m.confidence >= min_conf]
87+ done_rows.append((r["section_id"], r["accession_number"], backend, len(ms)))
88+ buf.extend(ms)
89+ if len(buf) >= flush_every:
90+ flush()
91+ if done % 2000 == 0:
92+ log.info(" ... %d/%d sections, %d mentions kept", done, len(rows), kept + len(buf))
93+ flush()
94+ log.info("Extraction complete: %d mentions from %d sections.", kept, len(rows))
95+ return {"sections": len(rows), "mentions": kept, "backend": backend}
added code/audit/spid_snapshot/resolve.py +75 −0
@@ -0,0 +1,75 @@
1+# ============================================================
2+# SPID — project resolution & timelines
3+#
4+# Merges many mentions (across filings/years) into a single
5+# Project record + a chronological timeline. Pure DuckDB SQL
6+# so it scales to millions of mentions.
7+# ============================================================
8+from __future__ import annotations
9+
10+from spid.config import get_logger
11+from spid.store import Store
12+
13+log = get_logger("resolve")
14+
15+# Grouping key: a project is identified by (company, type, anchor)
16+# where anchor = first location if present, else first salient name token.
17+_BUILD_PROJECTS = """
18+CREATE OR REPLACE TEMP TABLE _m AS
19+SELECT *,
20+ lower(coalesce(try(locations[1]), '')) AS loc1,
21+ regexp_extract(
22+ lower(regexp_replace(project_name, '[^A-Za-z ]', '', 'g')),
23+ '([a-z]{4,})', 1) AS name_tok
24+FROM project_mentions;
25+
26+CREATE OR REPLACE TEMP TABLE _keyed AS
27+SELECT *,
28+ cik || '|' || project_type || '|' ||
29+ CASE WHEN loc1 <> '' THEN loc1 ELSE coalesce(nullif(name_tok,''),'general') END AS gkey,
30+ md5(cik || '|' || project_type || '|' ||
31+ CASE WHEN loc1 <> '' THEN loc1 ELSE coalesce(nullif(name_tok,''),'general') END) AS project_id
32+FROM _m;
33+
34+DELETE FROM projects;
35+INSERT INTO projects
36+SELECT
37+ project_id,
38+ any_value(cik) AS cik,
39+ any_value(ticker) AS ticker,
40+ any_value(company_name) AS company_name,
41+ any_value(sector) AS sector,
42+ any_value(project_type) AS project_type,
43+ -- representative name: the highest-confidence mention's name
44+ arg_max(project_name, confidence) AS project_name,
45+ nullif(any_value(loc1), '') AS canonical_location,
46+ list_distinct(flatten(list(technologies))) AS technologies,
47+ -- latest-by-date status
48+ arg_max(status, filing_date) AS status,
49+ max(amount_usd) AS total_amount_usd,
50+ count(*) AS n_mentions,
51+ count(DISTINCT accession_number) AS n_filings,
52+ min(filing_date) AS first_seen,
53+ max(filing_date) AS last_seen,
54+ min(list_min(years)) AS first_year,
55+ max(list_max(years)) AS last_year,
56+ round(avg(confidence), 3) AS avg_confidence,
57+ arg_max(description, confidence) AS description
58+FROM _keyed
59+GROUP BY project_id;
60+
61+DELETE FROM project_timeline;
62+INSERT INTO project_timeline
63+SELECT project_id, accession_number, filing_date, form_type, status, amount_usd,
64+ confidence, left(description, 300) AS snippet
65+FROM _keyed
66+ORDER BY project_id, filing_date;
67+"""
68+
69+
70+def resolve_projects(store: Store) -> dict:
71+ store.con.execute(_BUILD_PROJECTS)
72+ n_proj = store.count("projects")
73+ n_tl = store.count("project_timeline")
74+ log.info("Resolved %d projects from mentions (%d timeline rows).", n_proj, n_tl)
75+ return {"projects": n_proj, "timeline_rows": n_tl}
added code/audit/spid_snapshot/rules.py +120 −0
@@ -0,0 +1,120 @@
1+# ============================================================
2+# SPID — deterministic rule-based extractor
3+#
4+# High-precision baseline that needs NO LLM. Scans section text
5+# sentence-by-sentence for project signals, classifies the type,
6+# and pulls money / location / tech / year / status.
7+# ============================================================
8+from __future__ import annotations
9+
10+import re
11+
12+from spid import taxonomy as tx
13+from spid.extract.base import ProjectMention
14+
15+# split into sentence-ish units; keep it cheap for 100k+ sections
16+_SENT_SPLIT = re.compile(r"(?<=[.;:!?])\s+(?=[A-Z(\"$])")
17+_WS = re.compile(r"\s+")
18+# generic "project-ish" gate so we skip boilerplate sentences fast
19+_PROJECT_GATE = re.compile(
20+ r"\b(project|initiative|program|invest|expan|construct|build|launch|develop|"
21+ r"deploy|implement|acqui|partner|facilit|capacity|capital|migrat|transform|"
22+ r"pipeline|plant|data ?cent|automat|sustainab|renewable|restructur)\w*",
23+ re.IGNORECASE,
24+)
25+_NAMED_PROJECT = re.compile(
26+ r"\b((?:Project|Program|Initiative)\s+[A-Z][A-Za-z0-9\-]+"
27+ r"|[A-Z][A-Za-z0-9\-]+\s+(?:Project|Program|Initiative))\b"
28+)
29+
30+
31+def _clean(s: str) -> str:
32+ return _WS.sub(" ", s).strip()
33+
34+
35+class RuleExtractor:
36+ name = "rules"
37+
38+ def extract(self, *, text: str, accession_number: str, cik: str,
39+ section_id: str, section_name: str) -> list[ProjectMention]:
40+ if not text:
41+ return []
42+ text = text[:600_000]
43+ out: list[ProjectMention] = []
44+ seen: set[str] = set()
45+
46+ for raw_sent in _SENT_SPLIT.split(text):
47+ sent = _clean(raw_sent)
48+ if not (20 <= len(sent) <= 700):
49+ continue
50+ if not _PROJECT_GATE.search(sent):
51+ continue
52+
53+ low = sent.lower()
54+ matched_types: list[str] = []
55+ for ptype, regexes in tx.TYPE_SIGNALS.items():
56+ if any(rx.search(low) for rx in regexes):
57+ matched_types.append(ptype)
58+ if not matched_types:
59+ continue
60+
61+ amount, amount_raw = tx.extract_money(sent)
62+ locations = tx.extract_locations(sent)
63+ techs = tx.extract_technologies(sent)
64+ years = tx.extract_years(sent)
65+ status = tx.detect_status(sent)
66+ nm = _NAMED_PROJECT.search(sent)
67+
68+ # keep at most the 2 strongest types for one sentence
69+ for ptype in matched_types[:2]:
70+ anchor = f"{ptype}:{(locations[0] if locations else '')}:{int(amount or 0)}"
71+ if anchor in seen:
72+ continue
73+ seen.add(anchor)
74+
75+ conf = self._confidence(ptype, amount, locations, years, status, nm, low)
76+ if conf < 0.30:
77+ continue
78+
79+ pname = nm.group(1) if nm else self._synth_name(ptype, locations, techs)
80+ out.append(ProjectMention(
81+ mention_id=ProjectMention.make_id(accession_number, section_id, ptype, anchor),
82+ accession_number=accession_number, cik=cik,
83+ section_id=section_id, section_name=section_name,
84+ project_name=pname, project_type=ptype,
85+ description=sent[:500],
86+ amount_usd=amount, amount_raw=amount_raw,
87+ locations=locations, technologies=techs, years=years,
88+ status=status, confidence=round(conf, 3), backend=self.name,
89+ ))
90+ return out
91+
92+ @staticmethod
93+ def _synth_name(ptype: str, locations: list[str], techs: list[str]) -> str:
94+ label = tx.TYPE_LABELS.get(ptype, ptype)
95+ if locations:
96+ return f"{label} — {locations[0]}"
97+ if techs:
98+ return f"{label} ({techs[0]})"
99+ return label
100+
101+ @staticmethod
102+ def _confidence(ptype, amount, locations, years, status, nm, low) -> float:
103+ c = 0.40
104+ if nm:
105+ c += 0.20
106+ if amount:
107+ c += 0.18
108+ if locations:
109+ c += 0.10
110+ if years:
111+ c += 0.06
112+ if status in ("planned", "in_progress", "completed"):
113+ c += 0.08
114+ # strong, specific phrasings
115+ if re.search(r"\b(announced|committed|invest\w+|will (build|open|invest))\b", low):
116+ c += 0.06
117+ # de-weight vague risk-factor / forward-looking hedges
118+ if re.search(r"\b(may|might|could|potential\w*|if we|uncertain)\b", low):
119+ c -= 0.08
120+ return max(0.0, min(1.0, c))
added code/audit/spid_snapshot/store.py +239 −0
@@ -0,0 +1,239 @@
1+# ============================================================
2+# SPID — DuckDB store
3+#
4+# Source corpus (filings / sections / companies) is exposed as
5+# read-only views over the existing parquet. SPID writes its
6+# own tables: mentions, projects, timeline, graph, embeddings.
7+# ============================================================
8+from __future__ import annotations
9+
10+from pathlib import Path
11+
12+import duckdb
13+
14+from spid.config import Settings, get_settings
15+from spid.extract.base import ProjectMention
16+
17+SCHEMA = """
18+CREATE TABLE IF NOT EXISTS project_mentions (
19+ mention_id VARCHAR PRIMARY KEY,
20+ accession_number VARCHAR,
21+ cik VARCHAR,
22+ section_id VARCHAR,
23+ section_name VARCHAR,
24+ project_name VARCHAR,
25+ project_type VARCHAR,
26+ description VARCHAR,
27+ objective VARCHAR,
28+ amount_usd DOUBLE,
29+ amount_raw VARCHAR,
30+ locations VARCHAR[],
31+ technologies VARCHAR[],
32+ partners VARCHAR[],
33+ suppliers VARCHAR[],
34+ benefits VARCHAR[],
35+ risks VARCHAR[],
36+ years INTEGER[],
37+ status VARCHAR,
38+ confidence DOUBLE,
39+ backend VARCHAR,
40+ -- denormalised filing context (filled on load)
41+ ticker VARCHAR,
42+ company_name VARCHAR,
43+ form_type VARCHAR,
44+ filing_date DATE,
45+ fiscal_year INTEGER,
46+ sector VARCHAR
47+);
48+
49+CREATE TABLE IF NOT EXISTS projects (
50+ project_id VARCHAR PRIMARY KEY,
51+ cik VARCHAR,
52+ ticker VARCHAR,
53+ company_name VARCHAR,
54+ sector VARCHAR,
55+ project_type VARCHAR,
56+ project_name VARCHAR,
57+ canonical_location VARCHAR,
58+ technologies VARCHAR[],
59+ status VARCHAR,
60+ total_amount_usd DOUBLE,
61+ n_mentions INTEGER,
62+ n_filings INTEGER,
63+ first_seen DATE,
64+ last_seen DATE,
65+ first_year INTEGER,
66+ last_year INTEGER,
67+ avg_confidence DOUBLE,
68+ description VARCHAR
69+);
70+
71+CREATE TABLE IF NOT EXISTS project_timeline (
72+ project_id VARCHAR,
73+ accession_number VARCHAR,
74+ filing_date DATE,
75+ form_type VARCHAR,
76+ status VARCHAR,
77+ amount_usd DOUBLE,
78+ confidence DOUBLE,
79+ snippet VARCHAR
80+);
81+
82+CREATE TABLE IF NOT EXISTS kg_nodes (
83+ node_id VARCHAR PRIMARY KEY,
84+ node_type VARCHAR, -- Company | Project | Technology | Location | Partner
85+ label VARCHAR,
86+ props JSON
87+);
88+
89+CREATE TABLE IF NOT EXISTS kg_edges (
90+ src VARCHAR,
91+ rel VARCHAR, -- owns | uses | located_in | partners_with | requires_capital
92+ dst VARCHAR,
93+ props JSON
94+);
95+
96+CREATE TABLE IF NOT EXISTS project_embeddings (
97+ project_id VARCHAR PRIMARY KEY,
98+ model VARCHAR,
99+ dim INTEGER,
100+ vector FLOAT[]
101+);
102+"""
103+
104+
105+class Store:
106+ def __init__(self, settings: Settings | None = None, read_only: bool = False):
107+ self.s = settings or get_settings()
108+ self.s.db_path.parent.mkdir(parents=True, exist_ok=True)
109+ self.con = duckdb.connect(str(self.s.db_path), read_only=read_only)
110+ if not read_only:
111+ self._init()
112+
113+ def _init(self) -> None:
114+ self.con.execute(
115+ "CREATE TABLE IF NOT EXISTS spid_sections ("
116+ "section_id VARCHAR PRIMARY KEY, accession_number VARCHAR, section_name VARCHAR,"
117+ "item_number VARCHAR, text VARCHAR, word_count INTEGER)")
118+ self._register_sources()
119+ self.con.execute(SCHEMA)
120+
121+ def _register_sources(self) -> None:
122+ c = self.con
123+ c.execute(
124+ "CREATE TABLE IF NOT EXISTS extracted_sections ("
125+ "section_id VARCHAR PRIMARY KEY, accession_number VARCHAR, backend VARCHAR,"
126+ "n_mentions INTEGER, extracted_at TIMESTAMP DEFAULT current_timestamp)")
127+ c.execute(f"CREATE OR REPLACE VIEW src_filings AS SELECT * FROM read_parquet('{self.s.filings_parquet}')")
128+ c.execute(f"CREATE OR REPLACE VIEW src_sections AS SELECT * FROM read_parquet('{self.s.sections_parquet}')")
129+ c.execute(f"CREATE OR REPLACE VIEW src_companies AS SELECT * FROM read_parquet('{self.s.companies_parquet}')")
130+ # unified, enriched section feed: pre-parsed sections (all forms since 2026-10-27)
131+ # UNION ingested 10-K items (legacy S&P 500 run); a 10-K already ingested in
132+ # spid_sections is not read a second time from the parsed corpus.
133+ scope = self.s.section_scope
134+ scope_sql = ""
135+ if scope:
136+ parts = []
137+ for form, items in scope.items():
138+ q = ",".join("'" + i + "'" for i in items)
139+ parts.append(f"(form_type = '{form}' AND section_name IN ({q}))")
140+ others = ",".join("'" + f + "'" for f in scope)
141+ scope_sql = " WHERE " + " OR ".join(parts) + f" OR form_type NOT IN ({others})"
142+ c.execute(f"""
143+ CREATE OR REPLACE VIEW v_sections AS
144+ WITH parsed AS (
145+ SELECT s.section_id, s.accession_number, s.section_name, s.item_number, s.text, s.word_count,
146+ f.form_type
147+ FROM src_sections s JOIN src_filings f USING (accession_number)
148+ WHERE NOT (f.form_type = '10-K'
149+ AND s.accession_number IN (SELECT DISTINCT accession_number FROM spid_sections))
150+ ),
151+ all_sec AS (
152+ SELECT section_id, accession_number, section_name, item_number, text, word_count
153+ FROM parsed{scope_sql}
154+ UNION ALL
155+ SELECT section_id, accession_number, section_name, item_number, text, word_count
156+ FROM spid_sections
157+ )
158+ SELECT s.section_id, s.accession_number, s.section_name, s.item_number,
159+ s.text, s.word_count,
160+ f.cik, f.ticker, f.company_name AS filing_company, f.form_type,
161+ CAST(f.filing_date AS DATE) AS filing_date, f.fiscal_year,
162+ co.company_name AS company_name, co.sector, co.sub_industry, co.sic_code, co.state
163+ FROM all_sec s
164+ JOIN src_filings f USING (accession_number)
165+ LEFT JOIN src_companies co ON f.cik = co.cik
166+ """)
167+
168+ # ---- read side ----------------------------------------------------
169+ def companies(self) -> list[tuple]:
170+ return self.con.execute(
171+ "SELECT DISTINCT cik, ticker FROM src_filings ORDER BY cik"
172+ ).fetchall()
173+
174+ def sections_for(self, *, ciks: list[str] | None = None, forms: list[str] | None = None,
175+ limit: int | None = None, min_words: int = 30, skip_done: bool = True) -> list[dict]:
176+ where = [f"word_count >= {int(min_words)}"]
177+ if skip_done:
178+ where.append("section_id NOT IN (SELECT section_id FROM extracted_sections)")
179+ if ciks:
180+ quoted = ",".join("'" + c.replace("'", "") + "'" for c in ciks)
181+ where.append(f"cik IN ({quoted})")
182+ if forms:
183+ quoted = ",".join("'" + f.replace("'", "") + "'" for f in forms)
184+ where.append(f"form_type IN ({quoted})")
185+ sql = "SELECT * FROM v_sections WHERE " + " AND ".join(where)
186+ sql += " ORDER BY filing_date"
187+ if limit:
188+ sql += f" LIMIT {int(limit)}"
189+ res = self.con.execute(sql)
190+ table = res.to_arrow_table() if hasattr(res, "to_arrow_table") else res.fetch_arrow_table() # DuckDB >= 2.0 / < 2.0
191+ return table.to_pylist()
192+
193+ # ---- write side ---------------------------------------------------
194+ def upsert_mentions(self, mentions: list[ProjectMention], ctx: dict) -> int:
195+ """ctx maps accession_number -> filing context dict."""
196+ if not mentions:
197+ return 0
198+ rows = []
199+ for m in mentions:
200+ cx = ctx.get(m.accession_number, {})
201+ rows.append((
202+ m.mention_id, m.accession_number, m.cik, m.section_id, m.section_name,
203+ m.project_name, m.project_type, m.description, m.objective,
204+ m.amount_usd, m.amount_raw, m.locations, m.technologies, m.partners,
205+ m.suppliers, m.benefits, m.risks, m.years, m.status, m.confidence, m.backend,
206+ cx.get("ticker"), cx.get("company_name") or cx.get("filing_company"),
207+ cx.get("form_type"), cx.get("filing_date"), cx.get("fiscal_year"), cx.get("sector"),
208+ ))
209+ self.con.executemany(
210+ "INSERT OR REPLACE INTO project_mentions VALUES "
211+ "(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", rows)
212+ return len(rows)
213+
214+ def mark_extracted(self, rows: list[tuple]) -> None:
215+ """rows: (section_id, accession_number, backend, n_mentions)."""
216+ if rows:
217+ self.con.executemany(
218+ "INSERT OR REPLACE INTO extracted_sections (section_id, accession_number, backend, n_mentions) "
219+ "VALUES (?,?,?,?)", rows)
220+
221+ def backfill_extracted(self, backend: str = "llm") -> int:
222+ """Record every section currently visible in v_sections as already extracted.
223+
224+ Used once, before the Russell 3000 corpus is merged, so the legacy S&P 500 run
225+ (whose sections were processed without a ledger) is not re-billed.
226+ """
227+ self.con.execute(f"""
228+ INSERT OR REPLACE INTO extracted_sections (section_id, accession_number, backend, n_mentions)
229+ SELECT v.section_id, v.accession_number, '{backend}',
230+ (SELECT COUNT(*) FROM project_mentions m WHERE m.section_id = v.section_id)
231+ FROM v_sections v
232+ """)
233+ return self.count("extracted_sections")
234+
235+ def count(self, table: str) -> int:
236+ return self.con.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]
237+
238+ def close(self) -> None:
239+ self.con.close()
added code/audit/spid_snapshot/taxonomy.py +261 −0
@@ -0,0 +1,261 @@
1+# ============================================================
2+# SPID — project taxonomy & lightweight value extractors
3+#
4+# Shared by both the rule-based and the LLM extractor so that
5+# `project_type`, `technology`, money, location and dates are
6+# normalised the same way regardless of backend.
7+# ============================================================
8+from __future__ import annotations
9+
10+import re
11+
12+# ----------------------------------------------------------------------
13+# Canonical project types. `signals` are high-precision regex fragments
14+# (already lower-cased) used by the rule extractor and as the controlled
15+# vocabulary the LLM is asked to classify into.
16+# ----------------------------------------------------------------------
17+PROJECT_TYPES: dict[str, dict] = {
18+ "manufacturing_expansion": {
19+ "label": "Manufacturing / Capacity Expansion",
20+ "signals": [
21+ r"(expand|increas\w+|add\w*|ramp\w*|scal\w+|new|build\w*|construct\w*|open\w*)"
22+ r"[^.]{0,60}(manufactur\w+|production|plant|factory|facilit\w+|capacity|assembly|fab\b)",
23+ ],
24+ },
25+ "plant_construction": {
26+ "label": "Plant / Site Construction",
27+ "signals": [r"(construct\w*|build\w*|broke ground|commission\w*|new (plant|site|facility))"
28+ r"[^.]{0,60}(plant|facility|site|campus|complex|refinery|mill)"],
29+ },
30+ "data_center": {
31+ "label": "Data Center",
32+ "signals": [r"data\s?cent(er|re)s?", r"hyperscal\w+", r"server farm"],
33+ },
34+ "cloud_migration": {
35+ "label": "Cloud Migration",
36+ "signals": [r"(migrat\w+|mov\w+|transition\w*)[^.]{0,40}(cloud)",
37+ r"cloud (migration|platform|adoption|infrastructure|first)"],
38+ },
39+ "ai_initiative": {
40+ "label": "AI / ML Initiative",
41+ "signals": [r"\b(artificial intelligence|machine learning|generative ai|gen ?ai|deep learning|"
42+ r"large language model|\bllm\b|neural network)\b",
43+ r"\bai[- ](powered|driven|enabled|capabilit\w+|platform|model|tools?)\b"],
44+ },
45+ "digital_transformation": {
46+ "label": "Digital Transformation",
47+ "signals": [r"digital (transformation|transition|initiative|strateg\w+|platform)",
48+ r"digitali[sz]ation"],
49+ },
50+ "erp_implementation": {
51+ "label": "ERP / Systems Implementation",
52+ "signals": [r"\b(erp|sap|oracle|workday|s/?4 ?hana)\b[^.]{0,40}(implement\w+|deploy\w+|migrat\w+|system|platform)",
53+ r"(implement\w+|deploy\w+|upgrad\w+)[^.]{0,40}\berp\b",
54+ r"enterprise resource planning"],
55+ },
56+ "rd_program": {
57+ "label": "R&D Program",
58+ "signals": [r"research and development", r"\br&d\b",
59+ r"(develop\w+|invest\w+)[^.]{0,40}(next[- ]generation|new technolog\w+|new product)"],
60+ },
61+ "product_launch": {
62+ "label": "New Product / Launch",
63+ "signals": [r"(launch\w*|introduc\w+|releas\w+|roll\w*[- ]?out|unveil\w+)"
64+ r"[^.]{0,50}(product|platform|service|offering|solution|model|line)",
65+ r"new product (line|launch|introduction)"],
66+ },
67+ "drug_pipeline": {
68+ "label": "Drug / Clinical Pipeline",
69+ "signals": [r"(phase\s?(i{1,3}|1|2|3)\b)", r"clinical trial", r"\bfda (approval|clearance|submission)\b",
70+ r"(drug|therap\w+|candidate|molecule|pipeline)[^.]{0,40}(develop\w+|trial|approval)"],
71+ },
72+ "ma_integration": {
73+ "label": "M&A / Integration",
74+ "signals": [r"(acqui\w+|merg\w+|takeover|business combination)",
75+ r"integration of [a-z]", r"purchase of [a-z]"],
76+ },
77+ "geographic_expansion": {
78+ "label": "Geographic Expansion",
79+ "signals": [r"(expand\w*|enter\w*|launch\w*|establish\w*)[^.]{0,50}"
80+ r"(market|region|countr\w+|geograph\w+|international|overseas|abroad)",
81+ r"market entry"],
82+ },
83+ "capex_program": {
84+ "label": "Capital Expenditure Program",
85+ "signals": [r"capital (expenditure|expenditures|investment|spending|program|project|budget)",
86+ r"\bcapex\b", r"invest\w+[^.]{0,30}(\$|usd)"],
87+ },
88+ "supply_chain": {
89+ "label": "Supply Chain Initiative",
90+ "signals": [r"supply chain", r"(reshor\w+|nearshor\w+|onshor\w+|offshor\w+)",
91+ r"(distribution|logistics|fulfillment|warehouse|inventory)[^.]{0,40}"
92+ r"(network|center|optimi[sz]\w+|expansion|investment)"],
93+ },
94+ "automation": {
95+ "label": "Automation / Robotics",
96+ "signals": [r"automat\w+", r"robot\w+", r"(industrial|warehouse) automation"],
97+ },
98+ "sustainability": {
99+ "label": "Sustainability / ESG",
100+ "signals": [r"sustainabilit\w+", r"\besg\b", r"net[- ]zero", r"carbon (neutral|reduction|footprint|emissions?)",
101+ r"decarboni[sz]\w+", r"emission reduction"],
102+ },
103+ "energy_transition": {
104+ "label": "Energy Transition / Renewables",
105+ "signals": [r"renewable (energy|power)", r"(solar|wind|hydrogen|geothermal|battery storage)[^.]{0,40}"
106+ r"(project|facility|farm|plant|capacity|investment)",
107+ r"clean energy", r"energy transition"],
108+ },
109+ "infrastructure": {
110+ "label": "Infrastructure Investment",
111+ "signals": [r"infrastructure (investment|project|upgrade|expansion|spending|build\w*)",
112+ r"(network|grid|pipeline|broadband|fiber) (expansion|build\w*|upgrade|deployment)"],
113+ },
114+ "cost_reduction": {
115+ "label": "Cost Reduction / Restructuring",
116+ "signals": [r"(cost (reduction|savings|cutting)|restructur\w+|efficiency (program|initiative)|"
117+ r"productivity (program|initiative)|streamlin\w+|optimi[sz]ation (program|initiative))"],
118+ },
119+ "partnership": {
120+ "label": "Strategic Partnership",
121+ "signals": [r"(strategic (partnership|alliance|collaboration)|joint venture|\bjv\b|"
122+ r"(partner\w+|collaborat\w+|team\w* up)[^.]{0,40}(with))"],
123+ },
124+ "technology_deployment": {
125+ "label": "Technology Deployment",
126+ "signals": [r"(deploy\w+|implement\w+|roll\w*[- ]?out|adopt\w+|install\w+)[^.]{0,50}"
127+ r"(technolog\w+|system|platform|software|solution|tool)"],
128+ },
129+}
130+
131+# ----------------------------------------------------------------------
132+# Technology vocabulary -> canonical tag
133+# ----------------------------------------------------------------------
134+TECHNOLOGY_TERMS: dict[str, str] = {
135+ r"\b(artificial intelligence|machine learning|generative ai|gen ?ai|deep learning|llm|neural)\b": "AI/ML",
136+ r"\bcloud\b": "Cloud",
137+ r"\b(erp|sap|oracle|workday|s/?4 ?hana|peoplesoft)\b": "ERP",
138+ r"\b(robot\w+|automation|cobots?)\b": "Robotics/Automation",
139+ r"\b(semiconductor|chip|wafer|fab|node\b|nanometer|foundry)\b": "Semiconductors",
140+ r"\b(solar|wind|renewable|hydrogen|geothermal)\b": "Renewable Energy",
141+ r"\b(battery|batteries|energy storage|lithium)\b": "Battery/Storage",
142+ r"\b(ev|electric vehicle|charging)\b": "Electric Vehicles",
143+ r"\b(blockchain|crypto|digital asset|tokeni[sz])\b": "Blockchain",
144+ r"\b(5g|broadband|fiber|fibre|network infrastructure)\b": "Connectivity",
145+ r"\b(data\s?cent(er|re)|hyperscale|server)\b": "Data Center",
146+ r"\b(cyber ?security|encryption|zero trust)\b": "Cybersecurity",
147+ r"\b(iot|internet of things|sensors?|edge computing)\b": "IoT/Edge",
148+ r"\b(quantum)\b": "Quantum",
149+ r"\b(biolog\w+|gene|mrna|crispr|cell therapy|molecul\w+)\b": "Biotech",
150+ r"\b(saas|software[- ]as[- ]a[- ]service|platform)\b": "Software/SaaS",
151+}
152+
153+# ----------------------------------------------------------------------
154+# Status keywords -> canonical status
155+# ----------------------------------------------------------------------
156+STATUS_TERMS: list[tuple[str, str]] = [
157+ (r"\b(completed|commission\w+|operational|in service|came online|opened|finalized)\b", "completed"),
158+ (r"\b(under construction|underway|in progress|ongoing|ramping|being (built|implemented))\b", "in_progress"),
159+ (r"\b(announc\w+|plan\w+|intend\w+|expect\w+|will (build|invest|launch|open)|propos\w+|"
160+ r"anticipat\w+|target\w+|by (20\d\d)|slated)\b", "planned"),
161+]
162+
163+# ----------------------------------------------------------------------
164+# Geography
165+# ----------------------------------------------------------------------
166+US_STATES = {
167+ "alabama","alaska","arizona","arkansas","california","colorado","connecticut","delaware",
168+ "florida","georgia","hawaii","idaho","illinois","indiana","iowa","kansas","kentucky",
169+ "louisiana","maine","maryland","massachusetts","michigan","minnesota","mississippi",
170+ "missouri","montana","nebraska","nevada","new hampshire","new jersey","new mexico",
171+ "new york","north carolina","north dakota","ohio","oklahoma","oregon","pennsylvania",
172+ "rhode island","south carolina","south dakota","tennessee","texas","utah","vermont",
173+ "virginia","washington","west virginia","wisconsin","wyoming",
174+}
175+COUNTRIES = {
176+ "united states","u.s.","usa","canada","mexico","china","india","japan","germany","france",
177+ "united kingdom","u.k.","ireland","brazil","singapore","south korea","korea","taiwan",
178+ "vietnam","thailand","malaysia","indonesia","australia","spain","italy","netherlands",
179+ "switzerland","sweden","poland","israel","saudi arabia","united arab emirates","uae",
180+ "philippines","czech republic","hungary","turkey","egypt","south africa","argentina","chile",
181+}
182+
183+_MONEY_RE = re.compile(
184+ r"(?:US)?\$\s?(\d[\d,]*\.?\d*)\s?(billion|bn|million|mm|mn|m\b|thousand|k\b|trillion)?",
185+ re.IGNORECASE,
186+)
187+_MULT = {
188+ "trillion": 1e12, "billion": 1e9, "bn": 1e9, "million": 1e6, "mm": 1e6, "mn": 1e6,
189+ "m": 1e6, "thousand": 1e3, "k": 1e3,
190+}
191+_YEAR_RE = re.compile(r"\b(?:fiscal\s+|fy\s?)?(20[0-3]\d|19[89]\d)\b", re.IGNORECASE)
192+_QUARTER_RE = re.compile(r"\bQ([1-4])\s?(20\d\d)\b", re.IGNORECASE)
193+_TECH_COMPILED = [(re.compile(p, re.IGNORECASE), tag) for p, tag in TECHNOLOGY_TERMS.items()]
194+_STATUS_COMPILED = [(re.compile(p, re.IGNORECASE), s) for p, s in STATUS_TERMS]
195+_CITY_STATE_RE = re.compile(r"\bin\s+([A-Z][a-zA-Z.]+(?:\s[A-Z][a-zA-Z.]+){0,2}),\s+([A-Z][a-zA-Z]+)")
196+
197+
198+def extract_money(text: str) -> tuple[float | None, str | None]:
199+ """Return (usd_amount, raw_span) for the largest explicit dollar figure."""
200+ best_val, best_raw = None, None
201+ for m in _MONEY_RE.finditer(text):
202+ try:
203+ num = float(m.group(1).replace(",", ""))
204+ except ValueError:
205+ continue
206+ unit = (m.group(2) or "").lower().strip()
207+ val = num * _MULT.get(unit, 1.0)
208+ if best_val is None or val > best_val:
209+ best_val, best_raw = val, m.group(0).strip()
210+ return best_val, best_raw
211+
212+
213+def extract_locations(text: str) -> list[str]:
214+ low = text.lower()
215+ found: list[str] = []
216+ for st in US_STATES:
217+ if re.search(rf"\b{re.escape(st)}\b", low):
218+ found.append(st.title())
219+ for co in COUNTRIES:
220+ if re.search(rf"\b{re.escape(co)}\b", low):
221+ found.append(co.upper() if len(co) <= 4 else co.title())
222+ for m in _CITY_STATE_RE.finditer(text):
223+ found.append(f"{m.group(1)}, {m.group(2)}")
224+ # de-dup preserving order
225+ seen, out = set(), []
226+ for f in found:
227+ k = f.lower()
228+ if k not in seen:
229+ seen.add(k); out.append(f)
230+ return out[:6]
231+
232+
233+def extract_years(text: str) -> list[int]:
234+ yrs = {int(y) for y in _YEAR_RE.findall(text)}
235+ for m in _QUARTER_RE.finditer(text):
236+ yrs.add(int(m.group(2)))
237+ return sorted(yrs)
238+
239+
240+def extract_technologies(text: str) -> list[str]:
241+ tags = []
242+ for rx, tag in _TECH_COMPILED:
243+ if rx.search(text) and tag not in tags:
244+ tags.append(tag)
245+ return tags
246+
247+
248+def detect_status(text: str) -> str:
249+ for rx, status in _STATUS_COMPILED:
250+ if rx.search(text):
251+ return status
252+ return "mentioned"
253+
254+
255+# pre-compile project-type signals
256+TYPE_SIGNALS = {
257+ ptype: [re.compile(sig, re.IGNORECASE) for sig in spec["signals"]]
258+ for ptype, spec in PROJECT_TYPES.items()
259+}
260+TYPE_LABELS = {k: v["label"] for k, v in PROJECT_TYPES.items()}
261+ALL_TYPES = list(PROJECT_TYPES.keys())
added results/n0_probe/manifest.json +27 −0
@@ -0,0 +1,27 @@
1+{
2+ "package": "n0_probe",
3+ "script": "/Users/simon-pierreboucher/firc0001_review/scripts/a00_probe_schema.py",
4+ "script_sha256": "b3bd5ca2a11782452a7d9c78b7f1ff63236fc60847ad4fc16914c1383f3b5a93",
5+ "started": "2026-11-05T14:10:08",
6+ "ended": "2026-11-05T14:10:17",
7+ "wall_seconds": 8.3,
8+ "node": "m3u96b",
9+ "python": "3.12.15",
10+ "duckdb": "2.0.0",
11+ "threads": 4,
12+ "memory_limit": "24GB",
13+ "seed": 20261105,
14+ "databases": {
15+ "/Users/simon-pierreboucher/UQO_RESEARCH_DB/project-db/data/db/spid.duckdb": {
16+ "bytes": 3155701760,
17+ "mtime": "2026-10-28T12:21:11.091389"
18+ },
19+ "/Users/simon-pierreboucher/UQO_RESEARCH_DB/data/duckdb/edgar_research.duckdb": {
20+ "bytes": 73500995584,
21+ "mtime": "2026-10-27T21:34:46.297884"
22+ }
23+ },
24+ "read_only": true,
25+ "row_counts": {},
26+ "notes": {}
27+}
\ No newline at end of file
added results/n0_probe/schema.json +937 −0
@@ -0,0 +1,937 @@
1+{
2+ "extracted_sections": [
3+ [
4+ "section_id",
5+ "VARCHAR"
6+ ],
7+ [
8+ "accession_number",
9+ "VARCHAR"
10+ ],
11+ [
12+ "backend",
13+ "VARCHAR"
14+ ],
15+ [
16+ "n_mentions",
17+ "INTEGER"
18+ ],
19+ [
20+ "extracted_at",
21+ "TIMESTAMP"
22+ ]
23+ ],
24+ "project_mentions": [
25+ [
26+ "mention_id",
27+ "VARCHAR"
28+ ],
29+ [
30+ "accession_number",
31+ "VARCHAR"
32+ ],
33+ [
34+ "cik",
35+ "VARCHAR"
36+ ],
37+ [
38+ "section_id",
39+ "VARCHAR"
40+ ],
41+ [
42+ "section_name",
43+ "VARCHAR"
44+ ],
45+ [
46+ "project_name",
47+ "VARCHAR"
48+ ],
49+ [
50+ "project_type",
51+ "VARCHAR"
52+ ],
53+ [
54+ "description",
55+ "VARCHAR"
56+ ],
57+ [
58+ "objective",
59+ "VARCHAR"
60+ ],
61+ [
62+ "amount_usd",
63+ "DOUBLE"
64+ ],
65+ [
66+ "amount_raw",
67+ "VARCHAR"
68+ ],
69+ [
70+ "locations",
71+ "VARCHAR[]"
72+ ],
73+ [
74+ "technologies",
75+ "VARCHAR[]"
76+ ],
77+ [
78+ "partners",
79+ "VARCHAR[]"
80+ ],
81+ [
82+ "suppliers",
83+ "VARCHAR[]"
84+ ],
85+ [
86+ "benefits",
87+ "VARCHAR[]"
88+ ],
89+ [
90+ "risks",
91+ "VARCHAR[]"
92+ ],
93+ [
94+ "years",
95+ "INTEGER[]"
96+ ],
97+ [
98+ "status",
99+ "VARCHAR"
100+ ],
101+ [
102+ "confidence",
103+ "DOUBLE"
104+ ],
105+ [
106+ "backend",
107+ "VARCHAR"
108+ ],
109+ [
110+ "ticker",
111+ "VARCHAR"
112+ ],
113+ [
114+ "company_name",
115+ "VARCHAR"
116+ ],
117+ [
118+ "form_type",
119+ "VARCHAR"
120+ ],
121+ [
122+ "filing_date",
123+ "DATE"
124+ ],
125+ [
126+ "fiscal_year",
127+ "INTEGER"
128+ ],
129+ [
130+ "sector",
131+ "VARCHAR"
132+ ]
133+ ],
134+ "projects": [
135+ [
136+ "project_id",
137+ "VARCHAR"
138+ ],
139+ [
140+ "cik",
141+ "VARCHAR"
142+ ],
143+ [
144+ "ticker",
145+ "VARCHAR"
146+ ],
147+ [
148+ "company_name",
149+ "VARCHAR"
150+ ],
151+ [
152+ "sector",
153+ "VARCHAR"
154+ ],
155+ [
156+ "project_type",
157+ "VARCHAR"
158+ ],
159+ [
160+ "project_name",
161+ "VARCHAR"
162+ ],
163+ [
164+ "canonical_location",
165+ "VARCHAR"
166+ ],
167+ [
168+ "technologies",
169+ "VARCHAR[]"
170+ ],
171+ [
172+ "status",
173+ "VARCHAR"
174+ ],
175+ [
176+ "total_amount_usd",
177+ "DOUBLE"
178+ ],
179+ [
180+ "n_mentions",
181+ "INTEGER"
182+ ],
183+ [
184+ "n_filings",
185+ "INTEGER"
186+ ],
187+ [
188+ "first_seen",
189+ "DATE"
190+ ],
191+ [
192+ "last_seen",
193+ "DATE"
194+ ],
195+ [
196+ "first_year",
197+ "INTEGER"
198+ ],
199+ [
200+ "last_year",
201+ "INTEGER"
202+ ],
203+ [
204+ "avg_confidence",
205+ "DOUBLE"
206+ ],
207+ [
208+ "description",
209+ "VARCHAR"
210+ ]
211+ ],
212+ "project_timeline": [
213+ [
214+ "project_id",
215+ "VARCHAR"
216+ ],
217+ [
218+ "accession_number",
219+ "VARCHAR"
220+ ],
221+ [
222+ "filing_date",
223+ "DATE"
224+ ],
225+ [
226+ "form_type",
227+ "VARCHAR"
228+ ],
229+ [
230+ "status",
231+ "VARCHAR"
232+ ],
233+ [
234+ "amount_usd",
235+ "DOUBLE"
236+ ],
237+ [
238+ "confidence",
239+ "DOUBLE"
240+ ],
241+ [
242+ "snippet",
243+ "VARCHAR"
244+ ]
245+ ],
246+ "spid_sections": [
247+ [
248+ "section_id",
249+ "VARCHAR"
250+ ],
251+ [
252+ "accession_number",
253+ "VARCHAR"
254+ ],
255+ [
256+ "section_name",
257+ "VARCHAR"
258+ ],
259+ [
260+ "item_number",
261+ "VARCHAR"
262+ ],
263+ [
264+ "text",
265+ "VARCHAR"
266+ ],
267+ [
268+ "word_count",
269+ "INTEGER"
270+ ]
271+ ],
272+ "v_sections": [
273+ [
274+ "section_id",
275+ "VARCHAR"
276+ ],
277+ [
278+ "accession_number",
279+ "VARCHAR"
280+ ],
281+ [
282+ "section_name",
283+ "VARCHAR"
284+ ],
285+ [
286+ "item_number",
287+ "VARCHAR"
288+ ],
289+ [
290+ "text",
291+ "VARCHAR"
292+ ],
293+ [
294+ "word_count",
295+ "INTEGER"
296+ ],
297+ [
298+ "cik",
299+ "VARCHAR"
300+ ],
301+ [
302+ "ticker",
303+ "VARCHAR"
304+ ],
305+ [
306+ "filing_company",
307+ "VARCHAR"
308+ ],
309+ [
310+ "form_type",
311+ "VARCHAR"
312+ ],
313+ [
314+ "filing_date",
315+ "DATE"
316+ ],
317+ [
318+ "fiscal_year",
319+ "INTEGER"
320+ ],
321+ [
322+ "company_name",
323+ "VARCHAR"
324+ ],
325+ [
326+ "sector",
327+ "VARCHAR"
328+ ],
329+ [
330+ "sub_industry",
331+ "VARCHAR"
332+ ],
333+ [
334+ "sic_code",
335+ "VARCHAR"
336+ ],
337+ [
338+ "state",
339+ "VARCHAR"
340+ ]
341+ ],
342+ "src_filings": [
343+ [
344+ "accession_number",
345+ "VARCHAR"
346+ ],
347+ [
348+ "cik",
349+ "VARCHAR"
350+ ],
351+ [
352+ "ticker",
353+ "VARCHAR"
354+ ],
355+ [
356+ "company_name",
357+ "VARCHAR"
358+ ],
359+ [
360+ "form_type",
361+ "VARCHAR"
362+ ],
363+ [
364+ "filing_date",
365+ "DATE"
366+ ],
367+ [
368+ "report_date",
369+ "DATE"
370+ ],
371+ [
372+ "fiscal_year",
373+ "INTEGER"
374+ ],
375+ [
376+ "fiscal_quarter",
377+ "INTEGER"
378+ ],
379+ [
380+ "sec_url",
381+ "VARCHAR"
382+ ],
383+ [
384+ "local_path",
385+ "VARCHAR"
386+ ],
387+ [
388+ "primary_document",
389+ "VARCHAR"
390+ ],
391+ [
392+ "file_size_bytes",
393+ "BIGINT"
394+ ],
395+ [
396+ "download_status",
397+ "VARCHAR"
398+ ],
399+ [
400+ "parse_status",
401+ "VARCHAR"
402+ ],
403+ [
404+ "created_at",
405+ "TIMESTAMP"
406+ ],
407+ [
408+ "updated_at",
409+ "TIMESTAMP"
410+ ]
411+ ],
412+ "src_sections": [
413+ [
414+ "section_id",
415+ "VARCHAR"
416+ ],
417+ [
418+ "accession_number",
419+ "VARCHAR"
420+ ],
421+ [
422+ "section_name",
423+ "VARCHAR"
424+ ],
425+ [
426+ "item_number",
427+ "VARCHAR"
428+ ],
429+ [
430+ "section_order",
431+ "INTEGER"
432+ ],
433+ [
434+ "text",
435+ "VARCHAR"
436+ ],
437+ [
438+ "word_count",
439+ "INTEGER"
440+ ],
441+ [
442+ "char_count",
443+ "INTEGER"
444+ ],
445+ [
446+ "created_at",
447+ "TIMESTAMP"
448+ ]
449+ ],
450+ "src_companies": [
451+ [
452+ "cik",
453+ "VARCHAR"
454+ ],
455+ [
456+ "ticker",
457+ "VARCHAR"
458+ ],
459+ [
460+ "company_name",
461+ "VARCHAR"
462+ ],
463+ [
464+ "sector",
465+ "VARCHAR"
466+ ],
467+ [
468+ "sub_industry",
469+ "VARCHAR"
470+ ],
471+ [
472+ "sic_code",
473+ "VARCHAR"
474+ ],
475+ [
476+ "state",
477+ "VARCHAR"
478+ ],
479+ [
480+ "date_added",
481+ "DATE"
482+ ],
483+ [
484+ "created_at",
485+ "TIMESTAMP"
486+ ],
487+ [
488+ "updated_at",
489+ "TIMESTAMP"
490+ ],
491+ [
492+ "in_sp500",
493+ "BOOLEAN"
494+ ],
495+ [
496+ "in_russell3000",
497+ "BOOLEAN"
498+ ],
499+ [
500+ "sic_description",
501+ "VARCHAR"
502+ ],
503+ [
504+ "state_of_incorporation",
505+ "VARCHAR"
506+ ],
507+ [
508+ "exchange",
509+ "VARCHAR"
510+ ],
511+ [
512+ "fiscal_year_end",
513+ "VARCHAR"
514+ ],
515+ [
516+ "russell_weight_pct",
517+ "DOUBLE"
518+ ],
519+ [
520+ "shard_node",
521+ "VARCHAR"
522+ ]
523+ ],
524+ "views": [
525+ [
526+ "src_companies",
527+ "CREATE VIEW src_companies AS SELECT * FROM read_parquet('/Users/simon-pierreboucher/UQO_RESEARCH_DB/data/parquet/companies/companies.parquet');"
528+ ],
529+ [
530+ "src_filings",
531+ "CREATE VIEW src_filings AS SELECT * FROM read_parquet('/Users/simon-pierreboucher/UQO_RESEARCH_DB/data/parquet/filings/filings.parquet');"
532+ ],
533+ [
534+ "src_sections",
535+ "CREATE VIEW src_sections AS SELECT * FROM read_parquet('/Users/simon-pierreboucher/UQO_RESEARCH_DB/data/parquet/filing_sections/filing_sections.parquet');"
536+ ],
537+ [
538+ "v_sections",
539+ "CREATE VIEW v_sections AS WITH parsed AS (SELECT s.section_id, s.accession_number, s.section_name, s.item_number, s.\"text\", s.word_count, f.form_type FROM src_sections AS s INNER JOIN src_filings AS f USING (accession_number) WHERE (NOT ((f.form_type = '10-K') AND (s.accession_number = ANY(SELECT DISTINCT accession_number FROM spid_sections))))), all_sec AS ((SELECT section_id, accession_number, section_name, item_number, \"text\", word_count FROM parsed WHERE (((form_type = '10-K') AND (section_name IN ('business', 'risk_factors', 'properties', 'md_and_a', 'quantitative_disclosures'))) OR (NOT (form_type IN ('10-K'))))) UNION ALL (SELECT section_id, accession_number, section_name, item_number, \"text\", word_count FROM spid_sections)) SELECT s.section_id, s.accession_number, s.section_name, s.item_number, s.\"text\", s.word_count, f.cik, f.ticker, f.company_name AS filing_company, f.form_type, CAST(f.filing_date AS DATE) AS filing_date, f.fiscal_year, co.company_name AS company_name, co.sector, co.sub_industry, co.sic_code, co.state FROM all_sec AS s INNER JOIN src_filings AS f USING (accession_number) LEFT JOIN src_companies AS co ON ((f.cik = co.cik));"
540+ ]
541+ ],
542+ "counts": {
543+ "extracted_sections": 1123098,
544+ "project_mentions": 308065,
545+ "projects": 121099,
546+ "project_timeline": 308065,
547+ "spid_sections": 27164
548+ },
549+ "ledger_words": [
550+ 1123098,
551+ 3605932059
552+ ],
553+ "parquet_files": [
554+ 1342283,
555+ 29.13085581
556+ ],
557+ "edgar_filings": [
558+ [
559+ "accession_number",
560+ "VARCHAR"
561+ ],
562+ [
563+ "cik",
564+ "VARCHAR"
565+ ],
566+ [
567+ "ticker",
568+ "VARCHAR"
569+ ],
570+ [
571+ "company_name",
572+ "VARCHAR"
573+ ],
574+ [
575+ "form_type",
576+ "VARCHAR"
577+ ],
578+ [
579+ "filing_date",
580+ "DATE"
581+ ],
582+ [
583+ "report_date",
584+ "DATE"
585+ ],
586+ [
587+ "fiscal_year",
588+ "INTEGER"
589+ ],
590+ [
591+ "fiscal_quarter",
592+ "INTEGER"
593+ ],
594+ [
595+ "sec_url",
596+ "VARCHAR"
597+ ],
598+ [
599+ "local_path",
600+ "VARCHAR"
601+ ],
602+ [
603+ "primary_document",
604+ "VARCHAR"
605+ ],
606+ [
607+ "file_size_bytes",
608+ "BIGINT"
609+ ],
610+ [
611+ "download_status",
612+ "VARCHAR"
613+ ],
614+ [
615+ "parse_status",
616+ "VARCHAR"
617+ ],
618+ [
619+ "created_at",
620+ "TIMESTAMP"
621+ ],
622+ [
623+ "updated_at",
624+ "TIMESTAMP"
625+ ]
626+ ],
627+ "edgar_filing_sections": [
628+ [
629+ "section_id",
630+ "VARCHAR"
631+ ],
632+ [
633+ "accession_number",
634+ "VARCHAR"
635+ ],
636+ [
637+ "section_name",
638+ "VARCHAR"
639+ ],
640+ [
641+ "item_number",
642+ "VARCHAR"
643+ ],
644+ [
645+ "section_order",
646+ "INTEGER"
647+ ],
648+ [
649+ "text",
650+ "VARCHAR"
651+ ],
652+ [
653+ "word_count",
654+ "INTEGER"
655+ ],
656+ [
657+ "char_count",
658+ "INTEGER"
659+ ],
660+ [
661+ "created_at",
662+ "TIMESTAMP"
663+ ]
664+ ],
665+ "edgar_companies": [
666+ [
667+ "cik",
668+ "VARCHAR"
669+ ],
670+ [
671+ "ticker",
672+ "VARCHAR"
673+ ],
674+ [
675+ "company_name",
676+ "VARCHAR"
677+ ],
678+ [
679+ "sector",
680+ "VARCHAR"
681+ ],
682+ [
683+ "sub_industry",
684+ "VARCHAR"
685+ ],
686+ [
687+ "sic_code",
688+ "VARCHAR"
689+ ],
690+ [
691+ "state",
692+ "VARCHAR"
693+ ],
694+ [
695+ "date_added",
696+ "DATE"
697+ ],
698+ [
699+ "created_at",
700+ "TIMESTAMP"
701+ ],
702+ [
703+ "updated_at",
704+ "TIMESTAMP"
705+ ],
706+ [
707+ "in_sp500",
708+ "BOOLEAN"
709+ ],
710+ [
711+ "in_russell3000",
712+ "BOOLEAN"
713+ ],
714+ [
715+ "sic_description",
716+ "VARCHAR"
717+ ],
718+ [
719+ "state_of_incorporation",
720+ "VARCHAR"
721+ ],
722+ [
723+ "exchange",
724+ "VARCHAR"
725+ ],
726+ [
727+ "fiscal_year_end",
728+ "VARCHAR"
729+ ],
730+ [
731+ "russell_weight_pct",
732+ "DOUBLE"
733+ ],
734+ [
735+ "shard_node",
736+ "VARCHAR"
737+ ]
738+ ],
739+ "edgar_annual_panel": [
740+ [
741+ "cik",
742+ "VARCHAR"
743+ ],
744+ [
745+ "ticker",
746+ "VARCHAR"
747+ ],
748+ [
749+ "fiscal_year",
750+ "INTEGER"
751+ ],
752+ [
753+ "filing_count",
754+ "BIGINT"
755+ ],
756+ [
757+ "total_chunks",
758+ "BIGINT"
759+ ],
760+ [
761+ "ai_adoption_mean",
762+ "DOUBLE"
763+ ],
764+ [
765+ "ai_adoption_max",
766+ "DOUBLE"
767+ ],
768+ [
769+ "ai_adoption_std",
770+ "DOUBLE"
771+ ],
772+ [
773+ "ai_adoption_top5pct",
774+ "DOUBLE"
775+ ],
776+ [
777+ "ai_adoption_wmean",
778+ "DOUBLE"
779+ ],
780+ [
781+ "ai_disruption_risk_mean",
782+ "DOUBLE"
783+ ],
784+ [
785+ "ai_disruption_risk_max",
786+ "DOUBLE"
787+ ],
788+ [
789+ "ai_disruption_risk_std",
790+ "DOUBLE"
791+ ],
792+ [
793+ "ai_disruption_risk_top5pct",
794+ "DOUBLE"
795+ ],
796+ [
797+ "ai_disruption_risk_wmean",
798+ "DOUBLE"
799+ ],
800+ [
801+ "ai_infrastructure_mean",
802+ "DOUBLE"
803+ ],
804+ [
805+ "ai_infrastructure_max",
806+ "DOUBLE"
807+ ],
808+ [
809+ "ai_infrastructure_std",
810+ "DOUBLE"
811+ ],
812+ [
813+ "ai_infrastructure_top5pct",
814+ "DOUBLE"
815+ ],
816+ [
817+ "ai_infrastructure_wmean",
818+ "DOUBLE"
819+ ],
820+ [
821+ "esg_commitment_mean",
822+ "DOUBLE"
823+ ],
824+ [
825+ "esg_commitment_max",
826+ "DOUBLE"
827+ ],
828+ [
829+ "esg_commitment_std",
830+ "DOUBLE"
831+ ],
832+ [
833+ "esg_commitment_top5pct",
834+ "DOUBLE"
835+ ],
836+ [
837+ "esg_commitment_wmean",
838+ "DOUBLE"
839+ ],
840+ [
841+ "climate_risk_mean",
842+ "DOUBLE"
843+ ],
844+ [
845+ "climate_risk_max",
846+ "DOUBLE"
847+ ],
848+ [
849+ "climate_risk_std",
850+ "DOUBLE"
851+ ],
852+ [
853+ "climate_risk_top5pct",
854+ "DOUBLE"
855+ ],
856+ [
857+ "climate_risk_wmean",
858+ "DOUBLE"
859+ ],
860+ [
861+ "cybersecurity_mean",
862+ "DOUBLE"
863+ ],
864+ [
865+ "cybersecurity_max",
866+ "DOUBLE"
867+ ],
868+ [
869+ "cybersecurity_std",
870+ "DOUBLE"
871+ ],
872+ [
873+ "cybersecurity_top5pct",
874+ "DOUBLE"
875+ ],
876+ [
877+ "cybersecurity_wmean",
878+ "DOUBLE"
879+ ],
880+ [
881+ "automation_labor_mean",
882+ "DOUBLE"
883+ ],
884+ [
885+ "automation_labor_max",
886+ "DOUBLE"
887+ ],
888+ [
889+ "automation_labor_std",
890+ "DOUBLE"
891+ ],
892+ [
893+ "automation_labor_top5pct",
894+ "DOUBLE"
895+ ],
896+ [
897+ "automation_labor_wmean",
898+ "DOUBLE"
899+ ],
900+ [
901+ "macro_risk_mean",
902+ "DOUBLE"
903+ ],
904+ [
905+ "macro_risk_max",
906+ "DOUBLE"
907+ ],
908+ [
909+ "macro_risk_std",
910+ "DOUBLE"
911+ ],
912+ [
913+ "macro_risk_top5pct",
914+ "DOUBLE"
915+ ],
916+ [
917+ "macro_risk_wmean",
918+ "DOUBLE"
919+ ]
920+ ],
921+ "edgar_tables": [
922+ "anchor_embeddings",
923+ "annual_final",
924+ "annual_panel",
925+ "chunk_scores",
926+ "companies",
927+ "embeddings_metadata",
928+ "filings",
929+ "filing_chunks",
930+ "filing_scores",
931+ "filing_sections",
932+ "processing_log",
933+ "quarterly_final",
934+ "quarterly_panel"
935+ ],
936+ "temp_ok": true
937+}
\ No newline at end of file
938
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere