SPB Git
spboucher.ai

spb/firc0001_uqo

Public

N0: audit infrastructure (read-only helpers, hash-verified 2324b58 snapshot, spid import shim, schema probe)

Stage D node batch N0 for FIRC-0001. sha256 of the nine snapshot files equals the node copies
(extract/llm.py, rules.py, base.py, taxonomy.py, resolve.py mtime 2026-06-17); both DuckDB files
open read_only=True; TEMP tables allowed; DuckDB temp_directory redirected to ~/firc0001_review/tmp.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Nov 5, 2026) parent 678d233 Browse files

23 changed files +3,242 −0

23 changed files
Unified Split
added code/audit/a00_probe_schema.py +27 −0
@@ -0,0 +1,27 @@
1 +"""N0 probe: open both DuckDB files read-only, describe the tables used by N1-N13."""
2 +import sys, json
3 +from pathlib import Path
4 +sys.path.insert(0, str(Path(__file__).resolve().parent))
5 +from common import *
6 +
7 +r = Run("n0_probe")
8 +con = connect(SPID_DB)
9 +out = {}
10 +for t in ["extracted_sections", "project_mentions", "projects", "project_timeline", "spid_sections", "v_sections", "src_filings", "src_sections", "src_companies"]:
11 + out[t] = [tuple(x[:2]) for x in con.execute(f"DESCRIBE {t}").fetchall()]
12 +out["views"] = con.execute("select view_name, sql from duckdb_views() where not internal").fetchall()
13 +out["counts"] = {t: con.execute(f"select count(*) from {t}").fetchone()[0] for t in ["extracted_sections", "project_mentions", "projects", "project_timeline", "spid_sections"]}
14 +out["ledger_words"] = con.execute("select count(*), sum(v.word_count) from extracted_sections e join (select section_id, word_count from src_sections union all select section_id, word_count from spid_sections) v using(section_id)").fetchone()
15 +out["parquet_files"] = con.execute("select count(*), sum(length(text))/1e9 from src_sections").fetchone()
16 +e = connect(EDGAR_DB)
17 +for t in ["filings", "filing_sections", "companies", "annual_panel"]:
18 + out["edgar_"+t] = [tuple(x[:2]) for x in e.execute(f"DESCRIBE {t}").fetchall()]
19 +out["edgar_tables"] = [x[0] for x in e.execute("select table_name from duckdb_tables() where not internal").fetchall()]
20 +r.write_json("schema.json", out)
21 +# can we create a temp table on a read_only connection?
22 +try:
23 + con.execute("CREATE TEMP TABLE _t AS SELECT 1 AS x"); out["temp_ok"] = True
24 +except Exception as ex:
25 + out["temp_ok"] = str(ex)
26 +r.write_json("schema.json", out)
27 +r.finish()
added code/audit/common.py +116 −0
@@ -0,0 +1,116 @@
1 +"""Shared helpers for the FIRC-0001 Stage D read-only audits (run on M3U96b).
2 +
3 +Every script opens the two DuckDB files with read_only=True, SET threads=4,
4 +SET memory_limit='24GB', writes CSV/JSON under ~/firc0001_review/results/<name>/
5 +and a manifest.json (script sha256, DuckDB mtimes, start/end, row counts).
6 +Nothing is ever written under ~/UQO_RESEARCH_DB.
7 +"""
8 +from __future__ import annotations
9 +
10 +import csv
11 +import datetime as dt
12 +import hashlib
13 +import json
14 +import os
15 +import sys
16 +import time
17 +from pathlib import Path
18 +
19 +import duckdb
20 +
21 +HOME = Path(os.environ.get("HOME", "~")).expanduser()
22 +RDB = HOME / "UQO_RESEARCH_DB"
23 +SPID_DB = RDB / "project-db" / "data" / "db" / "spid.duckdb"
24 +EDGAR_DB = RDB / "data" / "duckdb" / "edgar_research.duckdb"
25 +WORK = HOME / "firc0001_review"
26 +RESULTS = WORK / "results"
27 +THREADS = int(os.environ.get("FIRC_THREADS", "4"))
28 +MEMLIMIT = os.environ.get("FIRC_MEMLIMIT", "24GB")
29 +SEED = 20261105
30 +
31 +
32 +def connect(path: Path = SPID_DB) -> duckdb.DuckDBPyConnection:
33 + con = duckdb.connect(str(path), read_only=True)
34 + con.execute(f"SET threads={THREADS}")
35 + con.execute(f"SET memory_limit='{MEMLIMIT}'")
36 + con.execute("SET preserve_insertion_order=false")
37 + # spill files must never land next to the databases (default = <db>.tmp)
38 + (WORK / "tmp").mkdir(parents=True, exist_ok=True)
39 + con.execute(f"SET temp_directory='{WORK / 'tmp'}'")
40 + return con
41 +
42 +
43 +def attach_edgar(con: duckdb.DuckDBPyConnection, alias: str = "edgar") -> None:
44 + con.execute(f"ATTACH '{EDGAR_DB}' AS {alias} (READ_ONLY)")
45 +
46 +
47 +def sha256_file(p: Path) -> str:
48 + h = hashlib.sha256()
49 + with open(p, "rb") as f:
50 + for chunk in iter(lambda: f.read(1 << 20), b""):
51 + h.update(chunk)
52 + return h.hexdigest()
53 +
54 +
55 +class Run:
56 + """Context for one audit script: output dir, timing, manifest."""
57 +
58 + def __init__(self, name: str, script: str | None = None, out: str | None = None):
59 + self.name = name
60 + self.script = Path(script or sys.argv[0]).resolve()
61 + self.out = Path(out) if out else RESULTS / name
62 + self.out.mkdir(parents=True, exist_ok=True)
63 + self.t0 = time.time()
64 + self.started = dt.datetime.now().isoformat(timespec="seconds")
65 + self.rows: dict[str, int] = {}
66 + self.notes: dict = {}
67 +
68 + def write_csv(self, fname: str, cols: list[str], rows) -> int:
69 + rows = list(rows)
70 + with open(self.out / fname, "w", newline="") as f:
71 + w = csv.writer(f)
72 + w.writerow(cols)
73 + w.writerows(rows)
74 + self.rows[fname] = len(rows)
75 + print(f"[{self.name}] {fname}: {len(rows)} rows", flush=True)
76 + return len(rows)
77 +
78 + def dump(self, fname: str, con, sql: str, params=None):
79 + cur = con.execute(sql, params) if params else con.execute(sql)
80 + cols = [d[0] for d in cur.description]
81 + return self.write_csv(fname, cols, cur.fetchall())
82 +
83 + def write_json(self, fname: str, obj) -> None:
84 + with open(self.out / fname, "w") as f:
85 + json.dump(obj, f, indent=2, default=str)
86 + print(f"[{self.name}] {fname} written", flush=True)
87 +
88 + def finish(self, extra: dict | None = None) -> None:
89 + man = {
90 + "package": self.name,
91 + "script": str(self.script),
92 + "script_sha256": sha256_file(self.script) if self.script.exists() else None,
93 + "started": self.started,
94 + "ended": dt.datetime.now().isoformat(timespec="seconds"),
95 + "wall_seconds": round(time.time() - self.t0, 1),
96 + "node": os.uname().nodename,
97 + "python": sys.version.split()[0],
98 + "duckdb": duckdb.__version__,
99 + "threads": THREADS,
100 + "memory_limit": MEMLIMIT,
101 + "seed": SEED,
102 + "databases": {
103 + str(SPID_DB): {"bytes": SPID_DB.stat().st_size,
104 + "mtime": dt.datetime.fromtimestamp(SPID_DB.stat().st_mtime).isoformat()},
105 + str(EDGAR_DB): {"bytes": EDGAR_DB.stat().st_size,
106 + "mtime": dt.datetime.fromtimestamp(EDGAR_DB.stat().st_mtime).isoformat()},
107 + },
108 + "read_only": True,
109 + "row_counts": self.rows,
110 + "notes": self.notes,
111 + }
112 + if extra:
113 + man.update(extra)
114 + with open(self.out / "manifest.json", "w") as f:
115 + json.dump(man, f, indent=2, default=str)
116 + print(f"[{self.name}] done in {man['wall_seconds']} s", flush=True)
added code/audit/run_all.sh +12 −0
@@ -0,0 +1,12 @@
1 +#!/bin/bash
2 +# Stage D node batch for FIRC-0001: runs N1..N13 in the plan's order, sequentially, read-only.
3 +# usage: bash run_all.sh (on M3U96b, from ~/firc0001_review; scripts/ holds the aNN_*.py copies)
4 +set -u
5 +cd "$HOME/firc0001_review"
6 +ORDER="a01_ledger_reconciliation.py a03_resolver_speccurve.py a04_runs_segmenters.py a04_git_evidence.sh a02_grounding.py a05_focus_coverage.py a06_false_positive_classes.py a09_timeseries_cohort.py a10_universe_artefacts.py a07_8k_item_benchmark.py a08_rules_baseline.py a11_annotation_frame.py a12_env_lock.sh a13_export_stats_check.sh"
7 +for S in $ORDER; do
8 + bash scripts/run_one.sh "$S"
9 + NAME=$(basename "$S" | sed -E 's/\.(py|sh)$//')
10 + while [ ! -f "logs/$NAME.done" ]; do sleep 60; done
11 + cat "logs/$NAME.done"
12 +done
added code/audit/run_one.sh +13 −0
@@ -0,0 +1,13 @@
1 +#!/bin/bash
2 +# Run one audit script on M3U96b in the background under nice, with a log and a done-marker.
3 +# usage: run_one.sh <script.py|script.sh> [args...] (from ~/firc0001_review)
4 +set -u
5 +cd "$HOME/firc0001_review"
6 +S="$1"; shift
7 +NAME=$(basename "$S" | sed -E 's/\.(py|sh)$//')
8 +LOG="logs/$NAME.log"; DONE="logs/$NAME.done"
9 +rm -f "$DONE"
10 +PY="$HOME/UQO_RESEARCH_DB/.venv/bin/python"
11 +if [[ "$S" == *.sh ]]; then CMD="bash scripts/$S $*"; else CMD="$PY -I scripts/$S $*"; fi
12 +nohup bash -c "echo START \$(date -Iseconds); nice -n 10 env OMP_NUM_THREADS=4 FIRC_THREADS=4 $CMD; echo RC=\$? END \$(date -Iseconds) > $DONE" > "$LOG" 2>&1 &
13 +echo "started $NAME pid=$! log=$LOG"
added code/audit/spid/__init__.py +1 −0
@@ -0,0 +1 @@
1 +"""spid package shim: hash-verified snapshot copies of the 2324b58 extractor modules (see spid_snapshot/SHA256SUMS)."""
added code/audit/spid/config.py +126 −0
@@ -0,0 +1,126 @@
1 +# ============================================================
2 +# SPID — configuration & paths
3 +# ============================================================
4 +from __future__ import annotations
5 +
6 +import logging
7 +import os
8 +from dataclasses import dataclass, field
9 +from functools import lru_cache
10 +from pathlib import Path
11 +
12 +# project-db/ (this file lives in project-db/spid/config.py)
13 +PROJECT_ROOT = Path(__file__).resolve().parent.parent
14 +
15 +
16 +def _load_dotenv() -> None:
17 + """Minimal .env loader (no python-dotenv dependency required)."""
18 + env_path = PROJECT_ROOT / ".env"
19 + if not env_path.exists():
20 + return
21 + for line in env_path.read_text().splitlines():
22 + line = line.strip()
23 + if not line or line.startswith("#") or "=" not in line:
24 + continue
25 + key, _, val = line.partition("=")
26 + key, val = key.strip(), val.strip().strip('"').strip("'")
27 + os.environ.setdefault(key, val)
28 +
29 +
30 +_load_dotenv()
31 +# The parent UQO_RESEARCH_DB that holds the parsed corpus.
32 +RESEARCH_ROOT = PROJECT_ROOT.parent
33 +SOURCE_DATA = RESEARCH_ROOT / "data"
34 +
35 +
36 +def _parse_scope(spec: str) -> dict:
37 + out: dict = {}
38 + for part in (spec or "").split(";"):
39 + part = part.strip()
40 + if not part or ":" not in part:
41 + continue
42 + form, items = part.split(":", 1)
43 + out[form.strip()] = [i.strip() for i in items.split(",") if i.strip()]
44 + return out
45 +
46 +
47 +@dataclass(frozen=True)
48 +class Settings:
49 + """Resolved runtime configuration for the SPID platform."""
50 +
51 + # --- source corpus (read-only) -------------------------------------
52 + source_data: Path = SOURCE_DATA
53 + parquet_dir: Path = SOURCE_DATA / "parquet"
54 + processed_dir: Path = SOURCE_DATA / "processed"
55 +
56 + @property
57 + def filings_parquet(self) -> Path:
58 + return self.parquet_dir / "filings" / "filings.parquet"
59 +
60 + @property
61 + def sections_parquet(self) -> Path:
62 + return self.parquet_dir / "filing_sections" / "filing_sections.parquet"
63 +
64 + @property
65 + def companies_parquet(self) -> Path:
66 + return self.parquet_dir / "companies" / "companies.parquet"
67 +
68 + # --- SPID outputs (writable) ---------------------------------------
69 + db_path: Path = PROJECT_ROOT / "data" / "db" / "spid.duckdb"
70 + out_dir: Path = PROJECT_ROOT / "data"
71 +
72 + # --- extraction ----------------------------------------------------
73 + backend: str = field(
74 + default_factory=lambda: os.environ.get("SPID_BACKEND", "llm")
75 + ) # "llm" (gpt-4o-mini) | "rules"
76 + min_confidence: float = 0.45
77 + # which parsed sections feed the extractor, per form (None/empty = every section)
78 + section_scope: dict = field(default_factory=lambda: _parse_scope(os.environ.get(
79 + "SPID_SECTION_SCOPE",
80 + "10-K:business,risk_factors,properties,md_and_a,quantitative_disclosures")))
81 + max_section_chars: int = 600_000
82 +
83 + # --- embeddings ----------------------------------------------------
84 + embedding_model: str = field(
85 + default_factory=lambda: os.environ.get("SPID_EMBED_MODEL", "all-MiniLM-L6-v2")
86 + )
87 +
88 + # --- LLM backend (OpenAI-compatible) -------------------------------
89 + llm_base_url: str = field(
90 + default_factory=lambda: os.environ.get(
91 + "SPID_LLM_BASE_URL", "https://api.openai.com/v1"
92 + )
93 + )
94 + llm_model: str = field(
95 + default_factory=lambda: os.environ.get("SPID_LLM_MODEL", "gpt-4o-mini")
96 + )
97 + llm_api_key: str = field(
98 + default_factory=lambda: os.environ.get("OPENAI_API_KEY", "")
99 + or os.environ.get("MACLUSTR_API_KEY", "")
100 + )
101 + llm_timeout: float = 120.0
102 +
103 + def ensure_dirs(self) -> None:
104 + self.db_path.parent.mkdir(parents=True, exist_ok=True)
105 + (self.out_dir / "exports").mkdir(parents=True, exist_ok=True)
106 +
107 +
108 +@lru_cache(maxsize=1)
109 +def get_settings() -> Settings:
110 + s = Settings()
111 + s.ensure_dirs()
112 + return s
113 +
114 +
115 +def get_logger(name: str) -> logging.Logger:
116 + logger = logging.getLogger(f"spid.{name}")
117 + if not logger.handlers:
118 + h = logging.StreamHandler()
119 + h.setFormatter(
120 + logging.Formatter("%(asctime)s %(levelname)-7s %(name)s | %(message)s",
121 + datefmt="%H:%M:%S")
122 + )
123 + logger.addHandler(h)
124 + logger.setLevel(os.environ.get("SPID_LOG_LEVEL", "INFO"))
125 + logger.propagate = False
126 + return logger
added code/audit/spid/extract/__init__.py +0 −0
added code/audit/spid/extract/base.py +69 −0
@@ -0,0 +1,69 @@
1 +# ============================================================
2 +# SPID — extraction interface & record schema
3 +# ============================================================
4 +from __future__ import annotations
5 +
6 +import hashlib
7 +from dataclasses import dataclass, field, asdict
8 +from typing import Protocol
9 +
10 +
11 +@dataclass
12 +class ProjectMention:
13 + """A single project signal extracted from one filing section.
14 +
15 + Mentions are the atomic unit; `resolve.py` merges many mentions
16 + (across filings & years) into a single Project with a timeline.
17 + """
18 + mention_id: str
19 + accession_number: str
20 + cik: str
21 + section_id: str
22 + section_name: str
23 +
24 + project_name: str
25 + project_type: str
26 + description: str
27 + objective: str = ""
28 +
29 + amount_usd: float | None = None
30 + amount_raw: str | None = None
31 + locations: list[str] = field(default_factory=list)
32 + technologies: list[str] = field(default_factory=list)
33 + partners: list[str] = field(default_factory=list)
34 + suppliers: list[str] = field(default_factory=list)
35 + benefits: list[str] = field(default_factory=list)
36 + risks: list[str] = field(default_factory=list)
37 + years: list[int] = field(default_factory=list)
38 + status: str = "mentioned"
39 +
40 + confidence: float = 0.0
41 + backend: str = "rules"
42 +
43 + def to_row(self) -> dict:
44 + d = asdict(self)
45 + # store list fields as JSON-friendly via duckdb list[str]
46 + return d
47 +
48 + @staticmethod
49 + def make_id(accession: str, section_id: str, ptype: str, anchor: str) -> str:
50 + h = hashlib.sha1(f"{accession}|{section_id}|{ptype}|{anchor}".encode()).hexdigest()
51 + return h[:20]
52 +
53 +
54 +class Extractor(Protocol):
55 + name: str
56 +
57 + def extract(self, *, text: str, accession_number: str, cik: str,
58 + section_id: str, section_name: str) -> list[ProjectMention]:
59 + ...
60 +
61 +
62 +def get_extractor(backend: str):
63 + """Factory. `backend` in {'rules','llm'}."""
64 + backend = (backend or "rules").lower()
65 + if backend == "llm":
66 + from spid.extract.llm import LLMExtractor
67 + return LLMExtractor()
68 + from spid.extract.rules import RuleExtractor
69 + return RuleExtractor()
added code/audit/spid/extract/llm.py +202 −0
@@ -0,0 +1,202 @@
1 +# ============================================================
2 +# SPID — LLM extractor (OpenAI-compatible; default gpt-4o-mini)
3 +#
4 +# Uses the chat-completions JSON mode to extract structured
5 +# project records. Cost-controlled: only project-relevant
6 +# windows of each section are sent to the model.
7 +# ============================================================
8 +from __future__ import annotations
9 +
10 +import json
11 +import re
12 +import time
13 +
14 +import httpx
15 +
16 +from spid import taxonomy as tx
17 +from spid.config import get_logger, get_settings
18 +from spid.extract.base import ProjectMention
19 +from spid.extract.rules import _PROJECT_GATE
20 +
21 +log = get_logger("extract.llm")
22 +
23 +_SYSTEM = """You are a financial analyst extracting CORPORATE PROJECTS from SEC filing text.
24 +A "project" is any concrete initiative, investment, program, expansion, construction,
25 +product/R&D effort, acquisition/integration, technology deployment, partnership, or
26 +strategic program the company is pursuing.
27 +
28 +Return STRICT JSON: {"projects": [ ... ]}. Each project object has:
29 + project_name : short descriptive name (use the official name if one is stated)
30 + project_type : EXACTLY ONE of the allowed types below
31 + description : 1 sentence, grounded in the text (no speculation)
32 + objective : the goal/why (or "")
33 + status : one of planned | in_progress | completed | mentioned
34 + amount_usd : numeric USD figure if stated (e.g. 1500000000), else null
35 + locations : array of places (city/state/country), else []
36 + technologies : array of technologies involved, else []
37 + partners : array of named partners/JV counterparties, else []
38 + benefits : array of expected benefits, else []
39 + risks : array of stated risks, else []
40 + years : array of 4-digit years mentioned, else []
41 + confidence : 0.0-1.0 (how clearly this is a real, concrete project)
42 +
43 +Allowed project_type values:
44 +%s
45 +
46 +Rules:
47 +- Extract ONLY real initiatives, not generic risk-factor boilerplate or hypotheticals.
48 +- If the text describes no concrete project, return {"projects": []}.
49 +- Do not invent facts. Numbers/locations must appear in the text.
50 +- Prefer fewer high-quality projects over many vague ones.""" % "\n".join(
51 + f" - {k}: {v}" for k, v in tx.TYPE_LABELS.items()
52 +)
53 +
54 +
55 +def _focus(text: str, budget: int = 9000) -> str:
56 + """Keep only windows around project-signal hits to bound token cost."""
57 + if len(text) <= budget:
58 + return text
59 + spans: list[tuple[int, int]] = []
60 + for m in _PROJECT_GATE.finditer(text):
61 + spans.append((max(0, m.start() - 220), min(len(text), m.end() + 380)))
62 + if sum(b - a for a, b in spans) > budget * 1.4:
63 + break
64 + if not spans:
65 + return text[:budget]
66 + # merge overlapping windows
67 + spans.sort()
68 + merged = [spans[0]]
69 + for a, b in spans[1:]:
70 + if a <= merged[-1][1]:
71 + merged[-1] = (merged[-1][0], max(merged[-1][1], b))
72 + else:
73 + merged.append((a, b))
74 + chunks, total = [], 0
75 + for a, b in merged:
76 + chunks.append(text[a:b])
77 + total += b - a
78 + if total >= budget:
79 + break
80 + return "\n...\n".join(chunks)
81 +
82 +
83 +class LLMExtractor:
84 + name = "llm"
85 +
86 + def __init__(self) -> None:
87 + self.s = get_settings()
88 + if not self.s.llm_api_key:
89 + raise RuntimeError("No LLM API key configured (OPENAI_API_KEY / MACLUSTR_API_KEY).")
90 + self._client = httpx.Client(
91 + base_url=self.s.llm_base_url,
92 + headers={"Authorization": f"Bearer {self.s.llm_api_key}"},
93 + timeout=self.s.llm_timeout,
94 + )
95 +
96 + def _call(self, content: str, retries: int = 3) -> dict:
97 + payload = {
98 + "model": self.s.llm_model,
99 + "messages": [
100 + {"role": "system", "content": _SYSTEM},
101 + {"role": "user", "content": content},
102 + ],
103 + "temperature": 0,
104 + "response_format": {"type": "json_object"},
105 + "max_tokens": 2000,
106 + }
107 + last = None
108 + for attempt in range(retries):
109 + try:
110 + r = self._client.post("/chat/completions", json=payload)
111 + if r.status_code == 429 or r.status_code >= 500:
112 + raise httpx.HTTPStatusError("retryable", request=r.request, response=r)
113 + r.raise_for_status()
114 + txt = r.json()["choices"][0]["message"]["content"]
115 + return json.loads(txt)
116 + except (httpx.HTTPError, json.JSONDecodeError, KeyError) as e:
117 + last = e
118 + time.sleep(1.5 * (attempt + 1))
119 + log.warning("LLM call failed after %d retries: %s", retries, last)
120 + return {"projects": []}
121 +
122 + def extract(self, *, text: str, accession_number: str, cik: str,
123 + section_id: str, section_name: str) -> list[ProjectMention]:
124 + if not text or not _PROJECT_GATE.search(text):
125 + return []
126 + content = (f"SECTION: {section_name}\n\n" + _focus(text))[:14000]
127 + data = self._call(content)
128 + out: list[ProjectMention] = []
129 + for p in data.get("projects", []) or []:
130 + if not isinstance(p, dict):
131 + continue
132 + ptype = self._norm_type(p.get("project_type", ""))
133 + desc = str(p.get("description", "")).strip()[:600]
134 + if not desc:
135 + continue
136 + amount = p.get("amount_usd")
137 + try:
138 + amount = float(amount) if amount not in (None, "", "null") else None
139 + except (TypeError, ValueError):
140 + amount = None
141 + anchor = f"{ptype}:{str(p.get('project_name',''))[:40]}"
142 + conf = p.get("confidence", 0.6)
143 + try:
144 + conf = float(conf)
145 + except (TypeError, ValueError):
146 + conf = 0.6
147 + out.append(ProjectMention(
148 + mention_id=ProjectMention.make_id(accession_number, section_id, ptype, anchor),
149 + accession_number=accession_number, cik=cik,
150 + section_id=section_id, section_name=section_name,
151 + project_name=str(p.get("project_name") or tx.TYPE_LABELS.get(ptype, ptype))[:200],
152 + project_type=ptype, description=desc,
153 + objective=str(p.get("objective", ""))[:400],
154 + amount_usd=amount,
155 + amount_raw=(f"${amount:,.0f}" if amount else None),
156 + locations=_strlist(p.get("locations")),
157 + technologies=_strlist(p.get("technologies")) or tx.extract_technologies(desc),
158 + partners=_strlist(p.get("partners")),
159 + benefits=_strlist(p.get("benefits")),
160 + risks=_strlist(p.get("risks")),
161 + years=_intlist(p.get("years")) or tx.extract_years(desc),
162 + status=self._norm_status(p.get("status")),
163 + confidence=round(max(0.0, min(1.0, conf)), 3),
164 + backend=self.name,
165 + ))
166 + return out
167 +
168 + @staticmethod
169 + def _norm_type(raw: str) -> str:
170 + raw = (raw or "").strip().lower().replace(" ", "_").replace("-", "_")
171 + if raw in tx.PROJECT_TYPES:
172 + return raw
173 + # fuzzy contains match against labels/keys
174 + for k, label in tx.TYPE_LABELS.items():
175 + if raw and (raw in k or raw in label.lower().replace(" ", "_")):
176 + return k
177 + return "technology_deployment"
178 +
179 + @staticmethod
180 + def _norm_status(raw) -> str:
181 + raw = (str(raw or "")).strip().lower()
182 + return raw if raw in ("planned", "in_progress", "completed", "mentioned") else "mentioned"
183 +
184 +
185 +def _strlist(v) -> list[str]:
186 + if not v:
187 + return []
188 + if isinstance(v, str):
189 + v = [v]
190 + return [str(x).strip()[:120] for x in v if str(x).strip()][:8]
191 +
192 +
193 +def _intlist(v) -> list[int]:
194 + out = []
195 + for x in (v or []):
196 + try:
197 + y = int(str(x)[:4])
198 + if 1980 <= y <= 2040:
199 + out.append(y)
200 + except (TypeError, ValueError):
201 + continue
202 + return sorted(set(out))
added code/audit/spid/extract/rules.py +120 −0
@@ -0,0 +1,120 @@
1 +# ============================================================
2 +# SPID — deterministic rule-based extractor
3 +#
4 +# High-precision baseline that needs NO LLM. Scans section text
5 +# sentence-by-sentence for project signals, classifies the type,
6 +# and pulls money / location / tech / year / status.
7 +# ============================================================
8 +from __future__ import annotations
9 +
10 +import re
11 +
12 +from spid import taxonomy as tx
13 +from spid.extract.base import ProjectMention
14 +
15 +# split into sentence-ish units; keep it cheap for 100k+ sections
16 +_SENT_SPLIT = re.compile(r"(?<=[.;:!?])\s+(?=[A-Z(\"$])")
17 +_WS = re.compile(r"\s+")
18 +# generic "project-ish" gate so we skip boilerplate sentences fast
19 +_PROJECT_GATE = re.compile(
20 + r"\b(project|initiative|program|invest|expan|construct|build|launch|develop|"
21 + r"deploy|implement|acqui|partner|facilit|capacity|capital|migrat|transform|"
22 + r"pipeline|plant|data ?cent|automat|sustainab|renewable|restructur)\w*",
23 + re.IGNORECASE,
24 +)
25 +_NAMED_PROJECT = re.compile(
26 + r"\b((?:Project|Program|Initiative)\s+[A-Z][A-Za-z0-9\-]+"
27 + r"|[A-Z][A-Za-z0-9\-]+\s+(?:Project|Program|Initiative))\b"
28 +)
29 +
30 +
31 +def _clean(s: str) -> str:
32 + return _WS.sub(" ", s).strip()
33 +
34 +
35 +class RuleExtractor:
36 + name = "rules"
37 +
38 + def extract(self, *, text: str, accession_number: str, cik: str,
39 + section_id: str, section_name: str) -> list[ProjectMention]:
40 + if not text:
41 + return []
42 + text = text[:600_000]
43 + out: list[ProjectMention] = []
44 + seen: set[str] = set()
45 +
46 + for raw_sent in _SENT_SPLIT.split(text):
47 + sent = _clean(raw_sent)
48 + if not (20 <= len(sent) <= 700):
49 + continue
50 + if not _PROJECT_GATE.search(sent):
51 + continue
52 +
53 + low = sent.lower()
54 + matched_types: list[str] = []
55 + for ptype, regexes in tx.TYPE_SIGNALS.items():
56 + if any(rx.search(low) for rx in regexes):
57 + matched_types.append(ptype)
58 + if not matched_types:
59 + continue
60 +
61 + amount, amount_raw = tx.extract_money(sent)
62 + locations = tx.extract_locations(sent)
63 + techs = tx.extract_technologies(sent)
64 + years = tx.extract_years(sent)
65 + status = tx.detect_status(sent)
66 + nm = _NAMED_PROJECT.search(sent)
67 +
68 + # keep at most the 2 strongest types for one sentence
69 + for ptype in matched_types[:2]:
70 + anchor = f"{ptype}:{(locations[0] if locations else '')}:{int(amount or 0)}"
71 + if anchor in seen:
72 + continue
73 + seen.add(anchor)
74 +
75 + conf = self._confidence(ptype, amount, locations, years, status, nm, low)
76 + if conf < 0.30:
77 + continue
78 +
79 + pname = nm.group(1) if nm else self._synth_name(ptype, locations, techs)
80 + out.append(ProjectMention(
81 + mention_id=ProjectMention.make_id(accession_number, section_id, ptype, anchor),
82 + accession_number=accession_number, cik=cik,
83 + section_id=section_id, section_name=section_name,
84 + project_name=pname, project_type=ptype,
85 + description=sent[:500],
86 + amount_usd=amount, amount_raw=amount_raw,
87 + locations=locations, technologies=techs, years=years,
88 + status=status, confidence=round(conf, 3), backend=self.name,
89 + ))
90 + return out
91 +
92 + @staticmethod
93 + def _synth_name(ptype: str, locations: list[str], techs: list[str]) -> str:
94 + label = tx.TYPE_LABELS.get(ptype, ptype)
95 + if locations:
96 + return f"{label} — {locations[0]}"
97 + if techs:
98 + return f"{label} ({techs[0]})"
99 + return label
100 +
101 + @staticmethod
102 + def _confidence(ptype, amount, locations, years, status, nm, low) -> float:
103 + c = 0.40
104 + if nm:
105 + c += 0.20
106 + if amount:
107 + c += 0.18
108 + if locations:
109 + c += 0.10
110 + if years:
111 + c += 0.06
112 + if status in ("planned", "in_progress", "completed"):
113 + c += 0.08
114 + # strong, specific phrasings
115 + if re.search(r"\b(announced|committed|invest\w+|will (build|open|invest))\b", low):
116 + c += 0.06
117 + # de-weight vague risk-factor / forward-looking hedges
118 + if re.search(r"\b(may|might|could|potential\w*|if we|uncertain)\b", low):
119 + c -= 0.08
120 + return max(0.0, min(1.0, c))
added code/audit/spid/taxonomy.py +261 −0
@@ -0,0 +1,261 @@
1 +# ============================================================
2 +# SPID — project taxonomy & lightweight value extractors
3 +#
4 +# Shared by both the rule-based and the LLM extractor so that
5 +# `project_type`, `technology`, money, location and dates are
6 +# normalised the same way regardless of backend.
7 +# ============================================================
8 +from __future__ import annotations
9 +
10 +import re
11 +
12 +# ----------------------------------------------------------------------
13 +# Canonical project types. `signals` are high-precision regex fragments
14 +# (already lower-cased) used by the rule extractor and as the controlled
15 +# vocabulary the LLM is asked to classify into.
16 +# ----------------------------------------------------------------------
17 +PROJECT_TYPES: dict[str, dict] = {
18 + "manufacturing_expansion": {
19 + "label": "Manufacturing / Capacity Expansion",
20 + "signals": [
21 + r"(expand|increas\w+|add\w*|ramp\w*|scal\w+|new|build\w*|construct\w*|open\w*)"
22 + r"[^.]{0,60}(manufactur\w+|production|plant|factory|facilit\w+|capacity|assembly|fab\b)",
23 + ],
24 + },
25 + "plant_construction": {
26 + "label": "Plant / Site Construction",
27 + "signals": [r"(construct\w*|build\w*|broke ground|commission\w*|new (plant|site|facility))"
28 + r"[^.]{0,60}(plant|facility|site|campus|complex|refinery|mill)"],
29 + },
30 + "data_center": {
31 + "label": "Data Center",
32 + "signals": [r"data\s?cent(er|re)s?", r"hyperscal\w+", r"server farm"],
33 + },
34 + "cloud_migration": {
35 + "label": "Cloud Migration",
36 + "signals": [r"(migrat\w+|mov\w+|transition\w*)[^.]{0,40}(cloud)",
37 + r"cloud (migration|platform|adoption|infrastructure|first)"],
38 + },
39 + "ai_initiative": {
40 + "label": "AI / ML Initiative",
41 + "signals": [r"\b(artificial intelligence|machine learning|generative ai|gen ?ai|deep learning|"
42 + r"large language model|\bllm\b|neural network)\b",
43 + r"\bai[- ](powered|driven|enabled|capabilit\w+|platform|model|tools?)\b"],
44 + },
45 + "digital_transformation": {
46 + "label": "Digital Transformation",
47 + "signals": [r"digital (transformation|transition|initiative|strateg\w+|platform)",
48 + r"digitali[sz]ation"],
49 + },
50 + "erp_implementation": {
51 + "label": "ERP / Systems Implementation",
52 + "signals": [r"\b(erp|sap|oracle|workday|s/?4 ?hana)\b[^.]{0,40}(implement\w+|deploy\w+|migrat\w+|system|platform)",
53 + r"(implement\w+|deploy\w+|upgrad\w+)[^.]{0,40}\berp\b",
54 + r"enterprise resource planning"],
55 + },
56 + "rd_program": {
57 + "label": "R&D Program",
58 + "signals": [r"research and development", r"\br&d\b",
59 + r"(develop\w+|invest\w+)[^.]{0,40}(next[- ]generation|new technolog\w+|new product)"],
60 + },
61 + "product_launch": {
62 + "label": "New Product / Launch",
63 + "signals": [r"(launch\w*|introduc\w+|releas\w+|roll\w*[- ]?out|unveil\w+)"
64 + r"[^.]{0,50}(product|platform|service|offering|solution|model|line)",
65 + r"new product (line|launch|introduction)"],
66 + },
67 + "drug_pipeline": {
68 + "label": "Drug / Clinical Pipeline",
69 + "signals": [r"(phase\s?(i{1,3}|1|2|3)\b)", r"clinical trial", r"\bfda (approval|clearance|submission)\b",
70 + r"(drug|therap\w+|candidate|molecule|pipeline)[^.]{0,40}(develop\w+|trial|approval)"],
71 + },
72 + "ma_integration": {
73 + "label": "M&A / Integration",
74 + "signals": [r"(acqui\w+|merg\w+|takeover|business combination)",
75 + r"integration of [a-z]", r"purchase of [a-z]"],
76 + },
77 + "geographic_expansion": {
78 + "label": "Geographic Expansion",
79 + "signals": [r"(expand\w*|enter\w*|launch\w*|establish\w*)[^.]{0,50}"
80 + r"(market|region|countr\w+|geograph\w+|international|overseas|abroad)",
81 + r"market entry"],
82 + },
83 + "capex_program": {
84 + "label": "Capital Expenditure Program",
85 + "signals": [r"capital (expenditure|expenditures|investment|spending|program|project|budget)",
86 + r"\bcapex\b", r"invest\w+[^.]{0,30}(\$|usd)"],
87 + },
88 + "supply_chain": {
89 + "label": "Supply Chain Initiative",
90 + "signals": [r"supply chain", r"(reshor\w+|nearshor\w+|onshor\w+|offshor\w+)",
91 + r"(distribution|logistics|fulfillment|warehouse|inventory)[^.]{0,40}"
92 + r"(network|center|optimi[sz]\w+|expansion|investment)"],
93 + },
94 + "automation": {
95 + "label": "Automation / Robotics",
96 + "signals": [r"automat\w+", r"robot\w+", r"(industrial|warehouse) automation"],
97 + },
98 + "sustainability": {
99 + "label": "Sustainability / ESG",
100 + "signals": [r"sustainabilit\w+", r"\besg\b", r"net[- ]zero", r"carbon (neutral|reduction|footprint|emissions?)",
101 + r"decarboni[sz]\w+", r"emission reduction"],
102 + },
103 + "energy_transition": {
104 + "label": "Energy Transition / Renewables",
105 + "signals": [r"renewable (energy|power)", r"(solar|wind|hydrogen|geothermal|battery storage)[^.]{0,40}"
106 + r"(project|facility|farm|plant|capacity|investment)",
107 + r"clean energy", r"energy transition"],
108 + },
109 + "infrastructure": {
110 + "label": "Infrastructure Investment",
111 + "signals": [r"infrastructure (investment|project|upgrade|expansion|spending|build\w*)",
112 + r"(network|grid|pipeline|broadband|fiber) (expansion|build\w*|upgrade|deployment)"],
113 + },
114 + "cost_reduction": {
115 + "label": "Cost Reduction / Restructuring",
116 + "signals": [r"(cost (reduction|savings|cutting)|restructur\w+|efficiency (program|initiative)|"
117 + r"productivity (program|initiative)|streamlin\w+|optimi[sz]ation (program|initiative))"],
118 + },
119 + "partnership": {
120 + "label": "Strategic Partnership",
121 + "signals": [r"(strategic (partnership|alliance|collaboration)|joint venture|\bjv\b|"
122 + r"(partner\w+|collaborat\w+|team\w* up)[^.]{0,40}(with))"],
123 + },
124 + "technology_deployment": {
125 + "label": "Technology Deployment",
126 + "signals": [r"(deploy\w+|implement\w+|roll\w*[- ]?out|adopt\w+|install\w+)[^.]{0,50}"
127 + r"(technolog\w+|system|platform|software|solution|tool)"],
128 + },
129 +}
130 +
131 +# ----------------------------------------------------------------------
132 +# Technology vocabulary -> canonical tag
133 +# ----------------------------------------------------------------------
134 +TECHNOLOGY_TERMS: dict[str, str] = {
135 + r"\b(artificial intelligence|machine learning|generative ai|gen ?ai|deep learning|llm|neural)\b": "AI/ML",
136 + r"\bcloud\b": "Cloud",
137 + r"\b(erp|sap|oracle|workday|s/?4 ?hana|peoplesoft)\b": "ERP",
138 + r"\b(robot\w+|automation|cobots?)\b": "Robotics/Automation",
139 + r"\b(semiconductor|chip|wafer|fab|node\b|nanometer|foundry)\b": "Semiconductors",
140 + r"\b(solar|wind|renewable|hydrogen|geothermal)\b": "Renewable Energy",
141 + r"\b(battery|batteries|energy storage|lithium)\b": "Battery/Storage",
142 + r"\b(ev|electric vehicle|charging)\b": "Electric Vehicles",
143 + r"\b(blockchain|crypto|digital asset|tokeni[sz])\b": "Blockchain",
144 + r"\b(5g|broadband|fiber|fibre|network infrastructure)\b": "Connectivity",
145 + r"\b(data\s?cent(er|re)|hyperscale|server)\b": "Data Center",
146 + r"\b(cyber ?security|encryption|zero trust)\b": "Cybersecurity",
147 + r"\b(iot|internet of things|sensors?|edge computing)\b": "IoT/Edge",
148 + r"\b(quantum)\b": "Quantum",
149 + r"\b(biolog\w+|gene|mrna|crispr|cell therapy|molecul\w+)\b": "Biotech",
150 + r"\b(saas|software[- ]as[- ]a[- ]service|platform)\b": "Software/SaaS",
151 +}
152 +
153 +# ----------------------------------------------------------------------
154 +# Status keywords -> canonical status
155 +# ----------------------------------------------------------------------
156 +STATUS_TERMS: list[tuple[str, str]] = [
157 + (r"\b(completed|commission\w+|operational|in service|came online|opened|finalized)\b", "completed"),
158 + (r"\b(under construction|underway|in progress|ongoing|ramping|being (built|implemented))\b", "in_progress"),
159 + (r"\b(announc\w+|plan\w+|intend\w+|expect\w+|will (build|invest|launch|open)|propos\w+|"
160 + r"anticipat\w+|target\w+|by (20\d\d)|slated)\b", "planned"),
161 +]
162 +
163 +# ----------------------------------------------------------------------
164 +# Geography
165 +# ----------------------------------------------------------------------
166 +US_STATES = {
167 + "alabama","alaska","arizona","arkansas","california","colorado","connecticut","delaware",
168 + "florida","georgia","hawaii","idaho","illinois","indiana","iowa","kansas","kentucky",
169 + "louisiana","maine","maryland","massachusetts","michigan","minnesota","mississippi",
170 + "missouri","montana","nebraska","nevada","new hampshire","new jersey","new mexico",
171 + "new york","north carolina","north dakota","ohio","oklahoma","oregon","pennsylvania",
172 + "rhode island","south carolina","south dakota","tennessee","texas","utah","vermont",
173 + "virginia","washington","west virginia","wisconsin","wyoming",
174 +}
175 +COUNTRIES = {
176 + "united states","u.s.","usa","canada","mexico","china","india","japan","germany","france",
177 + "united kingdom","u.k.","ireland","brazil","singapore","south korea","korea","taiwan",
178 + "vietnam","thailand","malaysia","indonesia","australia","spain","italy","netherlands",
179 + "switzerland","sweden","poland","israel","saudi arabia","united arab emirates","uae",
180 + "philippines","czech republic","hungary","turkey","egypt","south africa","argentina","chile",
181 +}
182 +
183 +_MONEY_RE = re.compile(
184 + r"(?:US)?\$\s?(\d[\d,]*\.?\d*)\s?(billion|bn|million|mm|mn|m\b|thousand|k\b|trillion)?",
185 + re.IGNORECASE,
186 +)
187 +_MULT = {
188 + "trillion": 1e12, "billion": 1e9, "bn": 1e9, "million": 1e6, "mm": 1e6, "mn": 1e6,
189 + "m": 1e6, "thousand": 1e3, "k": 1e3,
190 +}
191 +_YEAR_RE = re.compile(r"\b(?:fiscal\s+|fy\s?)?(20[0-3]\d|19[89]\d)\b", re.IGNORECASE)
192 +_QUARTER_RE = re.compile(r"\bQ([1-4])\s?(20\d\d)\b", re.IGNORECASE)
193 +_TECH_COMPILED = [(re.compile(p, re.IGNORECASE), tag) for p, tag in TECHNOLOGY_TERMS.items()]
194 +_STATUS_COMPILED = [(re.compile(p, re.IGNORECASE), s) for p, s in STATUS_TERMS]
195 +_CITY_STATE_RE = re.compile(r"\bin\s+([A-Z][a-zA-Z.]+(?:\s[A-Z][a-zA-Z.]+){0,2}),\s+([A-Z][a-zA-Z]+)")
196 +
197 +
198 +def extract_money(text: str) -> tuple[float | None, str | None]:
199 + """Return (usd_amount, raw_span) for the largest explicit dollar figure."""
200 + best_val, best_raw = None, None
201 + for m in _MONEY_RE.finditer(text):
202 + try:
203 + num = float(m.group(1).replace(",", ""))
204 + except ValueError:
205 + continue
206 + unit = (m.group(2) or "").lower().strip()
207 + val = num * _MULT.get(unit, 1.0)
208 + if best_val is None or val > best_val:
209 + best_val, best_raw = val, m.group(0).strip()
210 + return best_val, best_raw
211 +
212 +
213 +def extract_locations(text: str) -> list[str]:
214 + low = text.lower()
215 + found: list[str] = []
216 + for st in US_STATES:
217 + if re.search(rf"\b{re.escape(st)}\b", low):
218 + found.append(st.title())
219 + for co in COUNTRIES:
220 + if re.search(rf"\b{re.escape(co)}\b", low):
221 + found.append(co.upper() if len(co) <= 4 else co.title())
222 + for m in _CITY_STATE_RE.finditer(text):
223 + found.append(f"{m.group(1)}, {m.group(2)}")
224 + # de-dup preserving order
225 + seen, out = set(), []
226 + for f in found:
227 + k = f.lower()
228 + if k not in seen:
229 + seen.add(k); out.append(f)
230 + return out[:6]
231 +
232 +
233 +def extract_years(text: str) -> list[int]:
234 + yrs = {int(y) for y in _YEAR_RE.findall(text)}
235 + for m in _QUARTER_RE.finditer(text):
236 + yrs.add(int(m.group(2)))
237 + return sorted(yrs)
238 +
239 +
240 +def extract_technologies(text: str) -> list[str]:
241 + tags = []
242 + for rx, tag in _TECH_COMPILED:
243 + if rx.search(text) and tag not in tags:
244 + tags.append(tag)
245 + return tags
246 +
247 +
248 +def detect_status(text: str) -> str:
249 + for rx, status in _STATUS_COMPILED:
250 + if rx.search(text):
251 + return status
252 + return "mentioned"
253 +
254 +
255 +# pre-compile project-type signals
256 +TYPE_SIGNALS = {
257 + ptype: [re.compile(sig, re.IGNORECASE) for sig in spec["signals"]]
258 + for ptype, spec in PROJECT_TYPES.items()
259 +}
260 +TYPE_LABELS = {k: v["label"] for k, v in PROJECT_TYPES.items()}
261 +ALL_TYPES = list(PROJECT_TYPES.keys())
added code/audit/spid_snapshot/SHA256SUMS +9 −0
@@ -0,0 +1,9 @@
1 +4ba1811f9513ab472855764ba8d09363060fed6e7d009f68f5b2749e85833ab1 worktrees/FIRC-0001/code/audit/spid_snapshot/base.py
2 +8c4a273631e0015867a1c88d92e284faca906ea16e9af2595dfa5cb06d5b8d06 worktrees/FIRC-0001/code/audit/spid_snapshot/config.py
3 +c53536becfe273f03882c309ba6ac79d4423c5bf1320865d044a0d00effada90 worktrees/FIRC-0001/code/audit/spid_snapshot/ingest.py
4 +ae7f3f61a8de36842ba8c99a566883465fe8d6672ec723ee438f963e50d4eb89 worktrees/FIRC-0001/code/audit/spid_snapshot/llm.py
5 +6135f39adb9e78b7e50b81a9034db07e45f30fcb3a329df5c6c9a79bad028712 worktrees/FIRC-0001/code/audit/spid_snapshot/pipeline.py
6 +925ab1258b2a364f1c9ad5be5817b4d815fe11e7fca5ad32f3ac71b5be59652d worktrees/FIRC-0001/code/audit/spid_snapshot/resolve.py
7 +7f191c18b0426d83931fa6f1f461f3a2d43bcc1d151a14a6e084f24db6766807 worktrees/FIRC-0001/code/audit/spid_snapshot/rules.py
8 +1219c923fff72b9946bc068baacd70802fd5fab86766e62141dbdd2ea066a54f worktrees/FIRC-0001/code/audit/spid_snapshot/store.py
9 +92ad69bcc4e448d03496943221fdcb78b4326baa7db682a6b6713c260c9c4e7a worktrees/FIRC-0001/code/audit/spid_snapshot/taxonomy.py
added code/audit/spid_snapshot/base.py +69 −0
@@ -0,0 +1,69 @@
1 +# ============================================================
2 +# SPID — extraction interface & record schema
3 +# ============================================================
4 +from __future__ import annotations
5 +
6 +import hashlib
7 +from dataclasses import dataclass, field, asdict
8 +from typing import Protocol
9 +
10 +
11 +@dataclass
12 +class ProjectMention:
13 + """A single project signal extracted from one filing section.
14 +
15 + Mentions are the atomic unit; `resolve.py` merges many mentions
16 + (across filings & years) into a single Project with a timeline.
17 + """
18 + mention_id: str
19 + accession_number: str
20 + cik: str
21 + section_id: str
22 + section_name: str
23 +
24 + project_name: str
25 + project_type: str
26 + description: str
27 + objective: str = ""
28 +
29 + amount_usd: float | None = None
30 + amount_raw: str | None = None
31 + locations: list[str] = field(default_factory=list)
32 + technologies: list[str] = field(default_factory=list)
33 + partners: list[str] = field(default_factory=list)
34 + suppliers: list[str] = field(default_factory=list)
35 + benefits: list[str] = field(default_factory=list)
36 + risks: list[str] = field(default_factory=list)
37 + years: list[int] = field(default_factory=list)
38 + status: str = "mentioned"
39 +
40 + confidence: float = 0.0
41 + backend: str = "rules"
42 +
43 + def to_row(self) -> dict:
44 + d = asdict(self)
45 + # store list fields as JSON-friendly via duckdb list[str]
46 + return d
47 +
48 + @staticmethod
49 + def make_id(accession: str, section_id: str, ptype: str, anchor: str) -> str:
50 + h = hashlib.sha1(f"{accession}|{section_id}|{ptype}|{anchor}".encode()).hexdigest()
51 + return h[:20]
52 +
53 +
54 +class Extractor(Protocol):
55 + name: str
56 +
57 + def extract(self, *, text: str, accession_number: str, cik: str,
58 + section_id: str, section_name: str) -> list[ProjectMention]:
59 + ...
60 +
61 +
62 +def get_extractor(backend: str):
63 + """Factory. `backend` in {'rules','llm'}."""
64 + backend = (backend or "rules").lower()
65 + if backend == "llm":
66 + from spid.extract.llm import LLMExtractor
67 + return LLMExtractor()
68 + from spid.extract.rules import RuleExtractor
69 + return RuleExtractor()
added code/audit/spid_snapshot/config.py +126 −0
@@ -0,0 +1,126 @@
1 +# ============================================================
2 +# SPID — configuration & paths
3 +# ============================================================
4 +from __future__ import annotations
5 +
6 +import logging
7 +import os
8 +from dataclasses import dataclass, field
9 +from functools import lru_cache
10 +from pathlib import Path
11 +
12 +# project-db/ (this file lives in project-db/spid/config.py)
13 +PROJECT_ROOT = Path(__file__).resolve().parent.parent
14 +
15 +
16 +def _load_dotenv() -> None:
17 + """Minimal .env loader (no python-dotenv dependency required)."""
18 + env_path = PROJECT_ROOT / ".env"
19 + if not env_path.exists():
20 + return
21 + for line in env_path.read_text().splitlines():
22 + line = line.strip()
23 + if not line or line.startswith("#") or "=" not in line:
24 + continue
25 + key, _, val = line.partition("=")
26 + key, val = key.strip(), val.strip().strip('"').strip("'")
27 + os.environ.setdefault(key, val)
28 +
29 +
30 +_load_dotenv()
31 +# The parent UQO_RESEARCH_DB that holds the parsed corpus.
32 +RESEARCH_ROOT = PROJECT_ROOT.parent
33 +SOURCE_DATA = RESEARCH_ROOT / "data"
34 +
35 +
36 +def _parse_scope(spec: str) -> dict:
37 + out: dict = {}
38 + for part in (spec or "").split(";"):
39 + part = part.strip()
40 + if not part or ":" not in part:
41 + continue
42 + form, items = part.split(":", 1)
43 + out[form.strip()] = [i.strip() for i in items.split(",") if i.strip()]
44 + return out
45 +
46 +
47 +@dataclass(frozen=True)
48 +class Settings:
49 + """Resolved runtime configuration for the SPID platform."""
50 +
51 + # --- source corpus (read-only) -------------------------------------
52 + source_data: Path = SOURCE_DATA
53 + parquet_dir: Path = SOURCE_DATA / "parquet"
54 + processed_dir: Path = SOURCE_DATA / "processed"
55 +
56 + @property
57 + def filings_parquet(self) -> Path:
58 + return self.parquet_dir / "filings" / "filings.parquet"
59 +
60 + @property
61 + def sections_parquet(self) -> Path:
62 + return self.parquet_dir / "filing_sections" / "filing_sections.parquet"
63 +
64 + @property
65 + def companies_parquet(self) -> Path:
66 + return self.parquet_dir / "companies" / "companies.parquet"
67 +
68 + # --- SPID outputs (writable) ---------------------------------------
69 + db_path: Path = PROJECT_ROOT / "data" / "db" / "spid.duckdb"
70 + out_dir: Path = PROJECT_ROOT / "data"
71 +
72 + # --- extraction ----------------------------------------------------
73 + backend: str = field(
74 + default_factory=lambda: os.environ.get("SPID_BACKEND", "llm")
75 + ) # "llm" (gpt-4o-mini) | "rules"
76 + min_confidence: float = 0.45
77 + # which parsed sections feed the extractor, per form (None/empty = every section)
78 + section_scope: dict = field(default_factory=lambda: _parse_scope(os.environ.get(
79 + "SPID_SECTION_SCOPE",
80 + "10-K:business,risk_factors,properties,md_and_a,quantitative_disclosures")))
81 + max_section_chars: int = 600_000
82 +
83 + # --- embeddings ----------------------------------------------------
84 + embedding_model: str = field(
85 + default_factory=lambda: os.environ.get("SPID_EMBED_MODEL", "all-MiniLM-L6-v2")
86 + )
87 +
88 + # --- LLM backend (OpenAI-compatible) -------------------------------
89 + llm_base_url: str = field(
90 + default_factory=lambda: os.environ.get(
91 + "SPID_LLM_BASE_URL", "https://api.openai.com/v1"
92 + )
93 + )
94 + llm_model: str = field(
95 + default_factory=lambda: os.environ.get("SPID_LLM_MODEL", "gpt-4o-mini")
96 + )
97 + llm_api_key: str = field(
98 + default_factory=lambda: os.environ.get("OPENAI_API_KEY", "")
99 + or os.environ.get("MACLUSTR_API_KEY", "")
100 + )
101 + llm_timeout: float = 120.0
102 +
103 + def ensure_dirs(self) -> None:
104 + self.db_path.parent.mkdir(parents=True, exist_ok=True)
105 + (self.out_dir / "exports").mkdir(parents=True, exist_ok=True)
106 +
107 +
108 +@lru_cache(maxsize=1)
109 +def get_settings() -> Settings:
110 + s = Settings()
111 + s.ensure_dirs()
112 + return s
113 +
114 +
115 +def get_logger(name: str) -> logging.Logger:
116 + logger = logging.getLogger(f"spid.{name}")
117 + if not logger.handlers:
118 + h = logging.StreamHandler()
119 + h.setFormatter(
120 + logging.Formatter("%(asctime)s %(levelname)-7s %(name)s | %(message)s",
121 + datefmt="%H:%M:%S")
122 + )
123 + logger.addHandler(h)
124 + logger.setLevel(os.environ.get("SPID_LOG_LEVEL", "INFO"))
125 + logger.propagate = False
126 + return logger
added code/audit/spid_snapshot/ingest.py +135 −0
@@ -0,0 +1,135 @@
1 +# ============================================================
2 +# SPID — raw 10-K ingester
3 +#
4 +# The pre-parsed corpus only sectioned 10-Q & 8-K filings; the
5 +# 10-K HTML was left raw. This module cleans the raw 10-K HTML
6 +# and extracts the project-dense Items (1 Business, 1A Risk,
7 +# 7 MD&A, 2 Properties) into a SPID-owned `spid_sections` table
8 +# that `v_sections` unions in — so 10-Ks flow through the same
9 +# extraction pipeline.
10 +# ============================================================
11 +from __future__ import annotations
12 +
13 +import glob
14 +import re
15 +import uuid
16 +from pathlib import Path
17 +
18 +from spid.config import get_logger, get_settings
19 +from spid.store import Store
20 +
21 +log = get_logger("ingest")
22 +
23 +RAW_SEC = get_settings().source_data / "raw" / "sec"
24 +
25 +# 10-K items we keep (where projects/capex/strategy concentrate)
26 +_KEEP_ITEMS = {
27 + "1": "business", "1A": "risk_factors", "2": "properties",
28 + "7": "md_and_a", "7A": "quantitative_disclosures",
29 +}
30 +_ITEM_RE = re.compile(r"(?:^|\n)\s*ITEM\s+(\d+[AB]?)\s*[.\-:—–]", re.IGNORECASE)
31 +_TAG_RE = re.compile(r"<[^>]+>")
32 +_WS_RE = re.compile(r"[ \t]+")
33 +_NL_RE = re.compile(r"\n{3,}")
34 +
35 +
36 +def _clean_html(path: Path) -> str:
37 + raw = path.read_text(errors="ignore")
38 + try:
39 + from bs4 import BeautifulSoup
40 + soup = BeautifulSoup(raw, "lxml")
41 + for t in soup(["script", "style", "table"]):
42 + t.decompose()
43 + text = soup.get_text("\n")
44 + except Exception: # noqa: BLE001
45 + text = _TAG_RE.sub(" ", raw)
46 + import html as _html
47 + text = _html.unescape(text)
48 + text = _WS_RE.sub(" ", text)
49 + text = _NL_RE.sub("\n\n", text)
50 + return text
51 +
52 +
53 +def _extract_items(text: str) -> list[tuple[str, str]]:
54 + matches = list(_ITEM_RE.finditer(text))
55 + if not matches:
56 + return [("full_body", text)]
57 + out: list[tuple[str, str]] = []
58 + for i, m in enumerate(matches):
59 + item = m.group(1).upper()
60 + if item not in _KEEP_ITEMS:
61 + continue
62 + start = m.start()
63 + end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
64 + body = text[start:end].strip()
65 + if len(body) > 400: # skip the table-of-contents stubs
66 + out.append((_KEEP_ITEMS[item], body[:600_000]))
67 + # keep the longest instance of each section (TOC vs real body)
68 + best: dict[str, str] = {}
69 + for name, body in out:
70 + if name not in best or len(body) > len(best[name]):
71 + best[name] = body
72 + return list(best.items())
73 +
74 +
75 +def _find_primary(acc: str, cik: str) -> Path | None:
76 + d = RAW_SEC / cik / "10-K" / acc.replace("-", "")
77 + if not d.is_dir():
78 + return None
79 + cands = glob.glob(str(d / "*.htm")) + glob.glob(str(d / "*.html")) + glob.glob(str(d / "*.txt"))
80 + cands = [c for c in cands if "R" + "" not in Path(c).name or True]
81 + if not cands:
82 + return None
83 + return Path(max(cands, key=lambda p: Path(p).stat().st_size))
84 +
85 +
86 +_SPID_SECTIONS_DDL = """
87 +CREATE TABLE IF NOT EXISTS spid_sections (
88 + section_id VARCHAR PRIMARY KEY,
89 + accession_number VARCHAR,
90 + section_name VARCHAR,
91 + item_number VARCHAR,
92 + text VARCHAR,
93 + word_count INTEGER
94 +);
95 +"""
96 +
97 +
98 +def ingest_10k(store: Store, *, ciks: list[str] | None = None, limit: int | None = None) -> dict:
99 + con = store.con
100 + con.execute(_SPID_SECTIONS_DDL)
101 + where = "form_type='10-K'"
102 + if ciks:
103 + q = ",".join("'" + c + "'" for c in ciks)
104 + where += f" AND cik IN ({q})"
105 + sql = f"SELECT accession_number, cik FROM src_filings WHERE {where} ORDER BY filing_date DESC"
106 + if limit:
107 + sql += f" LIMIT {int(limit)}"
108 + filings = con.execute(sql).fetchall()
109 +
110 + rows, done, ok = [], 0, 0
111 + for acc, cik in filings:
112 + done += 1
113 + # skip if already ingested
114 + if con.execute("SELECT 1 FROM spid_sections WHERE accession_number=? LIMIT 1", [acc]).fetchone():
115 + ok += 1
116 + continue
117 + path = _find_primary(acc, cik)
118 + if not path:
119 + continue
120 + try:
121 + text = _clean_html(path)
122 + for name, body in _extract_items(text):
123 + rows.append((str(uuid.uuid4()), acc, name, "", body, len(body.split())))
124 + ok += 1
125 + except Exception as e: # noqa: BLE001
126 + log.warning("ingest failed %s: %s", acc, e)
127 + if len(rows) >= 400:
128 + con.executemany("INSERT OR REPLACE INTO spid_sections VALUES (?,?,?,?,?,?)", rows); rows = []
129 + if done % 50 == 0:
130 + log.info(" ... ingested %d/%d 10-Ks", done, len(filings))
131 + if rows:
132 + con.executemany("INSERT OR REPLACE INTO spid_sections VALUES (?,?,?,?,?,?)", rows)
133 + n = store.count("spid_sections")
134 + log.info("Ingested 10-K sections: %d filings processed, %d section rows total.", ok, n)
135 + return {"filings_processed": ok, "section_rows": n}
added code/audit/spid_snapshot/llm.py +202 −0
@@ -0,0 +1,202 @@
1 +# ============================================================
2 +# SPID — LLM extractor (OpenAI-compatible; default gpt-4o-mini)
3 +#
4 +# Uses the chat-completions JSON mode to extract structured
5 +# project records. Cost-controlled: only project-relevant
6 +# windows of each section are sent to the model.
7 +# ============================================================
8 +from __future__ import annotations
9 +
10 +import json
11 +import re
12 +import time
13 +
14 +import httpx
15 +
16 +from spid import taxonomy as tx
17 +from spid.config import get_logger, get_settings
18 +from spid.extract.base import ProjectMention
19 +from spid.extract.rules import _PROJECT_GATE
20 +
21 +log = get_logger("extract.llm")
22 +
23 +_SYSTEM = """You are a financial analyst extracting CORPORATE PROJECTS from SEC filing text.
24 +A "project" is any concrete initiative, investment, program, expansion, construction,
25 +product/R&D effort, acquisition/integration, technology deployment, partnership, or
26 +strategic program the company is pursuing.
27 +
28 +Return STRICT JSON: {"projects": [ ... ]}. Each project object has:
29 + project_name : short descriptive name (use the official name if one is stated)
30 + project_type : EXACTLY ONE of the allowed types below
31 + description : 1 sentence, grounded in the text (no speculation)
32 + objective : the goal/why (or "")
33 + status : one of planned | in_progress | completed | mentioned
34 + amount_usd : numeric USD figure if stated (e.g. 1500000000), else null
35 + locations : array of places (city/state/country), else []
36 + technologies : array of technologies involved, else []
37 + partners : array of named partners/JV counterparties, else []
38 + benefits : array of expected benefits, else []
39 + risks : array of stated risks, else []
40 + years : array of 4-digit years mentioned, else []
41 + confidence : 0.0-1.0 (how clearly this is a real, concrete project)
42 +
43 +Allowed project_type values:
44 +%s
45 +
46 +Rules:
47 +- Extract ONLY real initiatives, not generic risk-factor boilerplate or hypotheticals.
48 +- If the text describes no concrete project, return {"projects": []}.
49 +- Do not invent facts. Numbers/locations must appear in the text.
50 +- Prefer fewer high-quality projects over many vague ones.""" % "\n".join(
51 + f" - {k}: {v}" for k, v in tx.TYPE_LABELS.items()
52 +)
53 +
54 +
55 +def _focus(text: str, budget: int = 9000) -> str:
56 + """Keep only windows around project-signal hits to bound token cost."""
57 + if len(text) <= budget:
58 + return text
59 + spans: list[tuple[int, int]] = []
60 + for m in _PROJECT_GATE.finditer(text):
61 + spans.append((max(0, m.start() - 220), min(len(text), m.end() + 380)))
62 + if sum(b - a for a, b in spans) > budget * 1.4:
63 + break
64 + if not spans:
65 + return text[:budget]
66 + # merge overlapping windows
67 + spans.sort()
68 + merged = [spans[0]]
69 + for a, b in spans[1:]:
70 + if a <= merged[-1][1]:
71 + merged[-1] = (merged[-1][0], max(merged[-1][1], b))
72 + else:
73 + merged.append((a, b))
74 + chunks, total = [], 0
75 + for a, b in merged:
76 + chunks.append(text[a:b])
77 + total += b - a
78 + if total >= budget:
79 + break
80 + return "\n...\n".join(chunks)
81 +
82 +
83 +class LLMExtractor:
84 + name = "llm"
85 +
86 + def __init__(self) -> None:
87 + self.s = get_settings()
88 + if not self.s.llm_api_key:
89 + raise RuntimeError("No LLM API key configured (OPENAI_API_KEY / MACLUSTR_API_KEY).")
90 + self._client = httpx.Client(
91 + base_url=self.s.llm_base_url,
92 + headers={"Authorization": f"Bearer {self.s.llm_api_key}"},
93 + timeout=self.s.llm_timeout,
94 + )
95 +
96 + def _call(self, content: str, retries: int = 3) -> dict:
97 + payload = {
98 + "model": self.s.llm_model,
99 + "messages": [
100 + {"role": "system", "content": _SYSTEM},
101 + {"role": "user", "content": content},
102 + ],
103 + "temperature": 0,
104 + "response_format": {"type": "json_object"},
105 + "max_tokens": 2000,
106 + }
107 + last = None
108 + for attempt in range(retries):
109 + try:
110 + r = self._client.post("/chat/completions", json=payload)
111 + if r.status_code == 429 or r.status_code >= 500:
112 + raise httpx.HTTPStatusError("retryable", request=r.request, response=r)
113 + r.raise_for_status()
114 + txt = r.json()["choices"][0]["message"]["content"]
115 + return json.loads(txt)
116 + except (httpx.HTTPError, json.JSONDecodeError, KeyError) as e:
117 + last = e
118 + time.sleep(1.5 * (attempt + 1))
119 + log.warning("LLM call failed after %d retries: %s", retries, last)
120 + return {"projects": []}
121 +
122 + def extract(self, *, text: str, accession_number: str, cik: str,
123 + section_id: str, section_name: str) -> list[ProjectMention]:
124 + if not text or not _PROJECT_GATE.search(text):
125 + return []
126 + content = (f"SECTION: {section_name}\n\n" + _focus(text))[:14000]
127 + data = self._call(content)
128 + out: list[ProjectMention] = []
129 + for p in data.get("projects", []) or []:
130 + if not isinstance(p, dict):
131 + continue
132 + ptype = self._norm_type(p.get("project_type", ""))
133 + desc = str(p.get("description", "")).strip()[:600]
134 + if not desc:
135 + continue
136 + amount = p.get("amount_usd")
137 + try:
138 + amount = float(amount) if amount not in (None, "", "null") else None
139 + except (TypeError, ValueError):
140 + amount = None
141 + anchor = f"{ptype}:{str(p.get('project_name',''))[:40]}"
142 + conf = p.get("confidence", 0.6)
143 + try:
144 + conf = float(conf)
145 + except (TypeError, ValueError):
146 + conf = 0.6
147 + out.append(ProjectMention(
148 + mention_id=ProjectMention.make_id(accession_number, section_id, ptype, anchor),
149 + accession_number=accession_number, cik=cik,
150 + section_id=section_id, section_name=section_name,
151 + project_name=str(p.get("project_name") or tx.TYPE_LABELS.get(ptype, ptype))[:200],
152 + project_type=ptype, description=desc,
153 + objective=str(p.get("objective", ""))[:400],
154 + amount_usd=amount,
155 + amount_raw=(f"${amount:,.0f}" if amount else None),
156 + locations=_strlist(p.get("locations")),
157 + technologies=_strlist(p.get("technologies")) or tx.extract_technologies(desc),
158 + partners=_strlist(p.get("partners")),
159 + benefits=_strlist(p.get("benefits")),
160 + risks=_strlist(p.get("risks")),
161 + years=_intlist(p.get("years")) or tx.extract_years(desc),
162 + status=self._norm_status(p.get("status")),
163 + confidence=round(max(0.0, min(1.0, conf)), 3),
164 + backend=self.name,
165 + ))
166 + return out
167 +
168 + @staticmethod
169 + def _norm_type(raw: str) -> str:
170 + raw = (raw or "").strip().lower().replace(" ", "_").replace("-", "_")
171 + if raw in tx.PROJECT_TYPES:
172 + return raw
173 + # fuzzy contains match against labels/keys
174 + for k, label in tx.TYPE_LABELS.items():
175 + if raw and (raw in k or raw in label.lower().replace(" ", "_")):
176 + return k
177 + return "technology_deployment"
178 +
179 + @staticmethod
180 + def _norm_status(raw) -> str:
181 + raw = (str(raw or "")).strip().lower()
182 + return raw if raw in ("planned", "in_progress", "completed", "mentioned") else "mentioned"
183 +
184 +
185 +def _strlist(v) -> list[str]:
186 + if not v:
187 + return []
188 + if isinstance(v, str):
189 + v = [v]
190 + return [str(x).strip()[:120] for x in v if str(x).strip()][:8]
191 +
192 +
193 +def _intlist(v) -> list[int]:
194 + out = []
195 + for x in (v or []):
196 + try:
197 + y = int(str(x)[:4])
198 + if 1980 <= y <= 2040:
199 + out.append(y)
200 + except (TypeError, ValueError):
201 + continue
202 + return sorted(set(out))
added code/audit/spid_snapshot/pipeline.py +95 −0
@@ -0,0 +1,95 @@
1 +# ============================================================
2 +# SPID — extraction engine
3 +#
4 +# Iterates enriched sections, runs the chosen extractor
5 +# (rules or gpt-4o-mini), filters by confidence, and loads
6 +# mentions into DuckDB. LLM calls are threaded (IO-bound).
7 +# ============================================================
8 +from __future__ import annotations
9 +
10 +from concurrent.futures import ThreadPoolExecutor, as_completed
11 +
12 +from spid.config import get_logger, get_settings
13 +from spid.extract.base import get_extractor
14 +from spid.store import Store
15 +
16 +log = get_logger("pipeline")
17 +
18 +
19 +def _ctx_from_rows(rows: list[dict]) -> dict:
20 + ctx = {}
21 + for r in rows:
22 + ctx.setdefault(r["accession_number"], {
23 + "ticker": r.get("ticker"),
24 + "company_name": r.get("company_name") or r.get("filing_company"),
25 + "filing_company": r.get("filing_company"),
26 + "form_type": r.get("form_type"),
27 + "filing_date": r.get("filing_date"),
28 + "fiscal_year": r.get("fiscal_year"),
29 + "sector": r.get("sector"),
30 + })
31 + return ctx
32 +
33 +
34 +def run_extraction(store: Store, *, backend: str | None = None,
35 + ciks: list[str] | None = None, forms: list[str] | None = None,
36 + limit_sections: int | None = None, min_words: int = 30,
37 + workers: int = 8, min_confidence: float | None = None,
38 + flush_every: int = 500) -> dict:
39 + s = get_settings()
40 + backend = backend or s.backend
41 + min_conf = s.min_confidence if min_confidence is None else min_confidence
42 + extractor = get_extractor(backend)
43 +
44 + rows = store.sections_for(ciks=ciks, forms=forms, limit=limit_sections, min_words=min_words)
45 + ctx = _ctx_from_rows(rows)
46 + log.info("Extracting from %d sections | backend=%s | workers=%d", len(rows), backend, workers)
47 +
48 + kept = 0
49 + buf = []
50 + done_rows: list[tuple] = []
51 +
52 + def work(r: dict):
53 + return extractor.extract(text=r["text"], accession_number=r["accession_number"],
54 + cik=r["cik"], section_id=r["section_id"],
55 + section_name=r["section_name"])
56 +
57 + def flush():
58 + nonlocal kept, buf, done_rows
59 + if buf:
60 + kept += store.upsert_mentions(buf, ctx)
61 + buf = []
62 + if done_rows:
63 + store.mark_extracted(done_rows)
64 + done_rows = []
65 +
66 + done = 0
67 + if backend == "llm" and workers > 1:
68 + with ThreadPoolExecutor(max_workers=workers) as ex:
69 + futs = {ex.submit(work, r): r for r in rows}
70 + for fut in as_completed(futs):
71 + done += 1
72 + r = futs[fut]
73 + try:
74 + ms = [m for m in fut.result() if m.confidence >= min_conf]
75 + done_rows.append((r["section_id"], r["accession_number"], backend, len(ms)))
76 + except Exception as e: # noqa: BLE001
77 + log.warning("section failed: %s", e); ms = []
78 + buf.extend(ms)
79 + if len(buf) >= flush_every:
80 + flush()
81 + if done % 200 == 0:
82 + log.info(" ... %d/%d sections, %d mentions kept", done, len(rows), kept + len(buf))
83 + else:
84 + for r in rows:
85 + done += 1
86 + ms = [m for m in work(r) if m.confidence >= min_conf]
87 + done_rows.append((r["section_id"], r["accession_number"], backend, len(ms)))
88 + buf.extend(ms)
89 + if len(buf) >= flush_every:
90 + flush()
91 + if done % 2000 == 0:
92 + log.info(" ... %d/%d sections, %d mentions kept", done, len(rows), kept + len(buf))
93 + flush()
94 + log.info("Extraction complete: %d mentions from %d sections.", kept, len(rows))
95 + return {"sections": len(rows), "mentions": kept, "backend": backend}
added code/audit/spid_snapshot/resolve.py +75 −0
@@ -0,0 +1,75 @@
1 +# ============================================================
2 +# SPID — project resolution & timelines
3 +#
4 +# Merges many mentions (across filings/years) into a single
5 +# Project record + a chronological timeline. Pure DuckDB SQL
6 +# so it scales to millions of mentions.
7 +# ============================================================
8 +from __future__ import annotations
9 +
10 +from spid.config import get_logger
11 +from spid.store import Store
12 +
13 +log = get_logger("resolve")
14 +
15 +# Grouping key: a project is identified by (company, type, anchor)
16 +# where anchor = first location if present, else first salient name token.
17 +_BUILD_PROJECTS = """
18 +CREATE OR REPLACE TEMP TABLE _m AS
19 +SELECT *,
20 + lower(coalesce(try(locations[1]), '')) AS loc1,
21 + regexp_extract(
22 + lower(regexp_replace(project_name, '[^A-Za-z ]', '', 'g')),
23 + '([a-z]{4,})', 1) AS name_tok
24 +FROM project_mentions;
25 +
26 +CREATE OR REPLACE TEMP TABLE _keyed AS
27 +SELECT *,
28 + cik || '|' || project_type || '|' ||
29 + CASE WHEN loc1 <> '' THEN loc1 ELSE coalesce(nullif(name_tok,''),'general') END AS gkey,
30 + md5(cik || '|' || project_type || '|' ||
31 + CASE WHEN loc1 <> '' THEN loc1 ELSE coalesce(nullif(name_tok,''),'general') END) AS project_id
32 +FROM _m;
33 +
34 +DELETE FROM projects;
35 +INSERT INTO projects
36 +SELECT
37 + project_id,
38 + any_value(cik) AS cik,
39 + any_value(ticker) AS ticker,
40 + any_value(company_name) AS company_name,
41 + any_value(sector) AS sector,
42 + any_value(project_type) AS project_type,
43 + -- representative name: the highest-confidence mention's name
44 + arg_max(project_name, confidence) AS project_name,
45 + nullif(any_value(loc1), '') AS canonical_location,
46 + list_distinct(flatten(list(technologies))) AS technologies,
47 + -- latest-by-date status
48 + arg_max(status, filing_date) AS status,
49 + max(amount_usd) AS total_amount_usd,
50 + count(*) AS n_mentions,
51 + count(DISTINCT accession_number) AS n_filings,
52 + min(filing_date) AS first_seen,
53 + max(filing_date) AS last_seen,
54 + min(list_min(years)) AS first_year,
55 + max(list_max(years)) AS last_year,
56 + round(avg(confidence), 3) AS avg_confidence,
57 + arg_max(description, confidence) AS description
58 +FROM _keyed
59 +GROUP BY project_id;
60 +
61 +DELETE FROM project_timeline;
62 +INSERT INTO project_timeline
63 +SELECT project_id, accession_number, filing_date, form_type, status, amount_usd,
64 + confidence, left(description, 300) AS snippet
65 +FROM _keyed
66 +ORDER BY project_id, filing_date;
67 +"""
68 +
69 +
70 +def resolve_projects(store: Store) -> dict:
71 + store.con.execute(_BUILD_PROJECTS)
72 + n_proj = store.count("projects")
73 + n_tl = store.count("project_timeline")
74 + log.info("Resolved %d projects from mentions (%d timeline rows).", n_proj, n_tl)
75 + return {"projects": n_proj, "timeline_rows": n_tl}
added code/audit/spid_snapshot/rules.py +120 −0
@@ -0,0 +1,120 @@
1 +# ============================================================
2 +# SPID — deterministic rule-based extractor
3 +#
4 +# High-precision baseline that needs NO LLM. Scans section text
5 +# sentence-by-sentence for project signals, classifies the type,
6 +# and pulls money / location / tech / year / status.
7 +# ============================================================
8 +from __future__ import annotations
9 +
10 +import re
11 +
12 +from spid import taxonomy as tx
13 +from spid.extract.base import ProjectMention
14 +
15 +# split into sentence-ish units; keep it cheap for 100k+ sections
16 +_SENT_SPLIT = re.compile(r"(?<=[.;:!?])\s+(?=[A-Z(\"$])")
17 +_WS = re.compile(r"\s+")
18 +# generic "project-ish" gate so we skip boilerplate sentences fast
19 +_PROJECT_GATE = re.compile(
20 + r"\b(project|initiative|program|invest|expan|construct|build|launch|develop|"
21 + r"deploy|implement|acqui|partner|facilit|capacity|capital|migrat|transform|"
22 + r"pipeline|plant|data ?cent|automat|sustainab|renewable|restructur)\w*",
23 + re.IGNORECASE,
24 +)
25 +_NAMED_PROJECT = re.compile(
26 + r"\b((?:Project|Program|Initiative)\s+[A-Z][A-Za-z0-9\-]+"
27 + r"|[A-Z][A-Za-z0-9\-]+\s+(?:Project|Program|Initiative))\b"
28 +)
29 +
30 +
31 +def _clean(s: str) -> str:
32 + return _WS.sub(" ", s).strip()
33 +
34 +
35 +class RuleExtractor:
36 + name = "rules"
37 +
38 + def extract(self, *, text: str, accession_number: str, cik: str,
39 + section_id: str, section_name: str) -> list[ProjectMention]:
40 + if not text:
41 + return []
42 + text = text[:600_000]
43 + out: list[ProjectMention] = []
44 + seen: set[str] = set()
45 +
46 + for raw_sent in _SENT_SPLIT.split(text):
47 + sent = _clean(raw_sent)
48 + if not (20 <= len(sent) <= 700):
49 + continue
50 + if not _PROJECT_GATE.search(sent):
51 + continue
52 +
53 + low = sent.lower()
54 + matched_types: list[str] = []
55 + for ptype, regexes in tx.TYPE_SIGNALS.items():
56 + if any(rx.search(low) for rx in regexes):
57 + matched_types.append(ptype)
58 + if not matched_types:
59 + continue
60 +
61 + amount, amount_raw = tx.extract_money(sent)
62 + locations = tx.extract_locations(sent)
63 + techs = tx.extract_technologies(sent)
64 + years = tx.extract_years(sent)
65 + status = tx.detect_status(sent)
66 + nm = _NAMED_PROJECT.search(sent)
67 +
68 + # keep at most the 2 strongest types for one sentence
69 + for ptype in matched_types[:2]:
70 + anchor = f"{ptype}:{(locations[0] if locations else '')}:{int(amount or 0)}"
71 + if anchor in seen:
72 + continue
73 + seen.add(anchor)
74 +
75 + conf = self._confidence(ptype, amount, locations, years, status, nm, low)
76 + if conf < 0.30:
77 + continue
78 +
79 + pname = nm.group(1) if nm else self._synth_name(ptype, locations, techs)
80 + out.append(ProjectMention(
81 + mention_id=ProjectMention.make_id(accession_number, section_id, ptype, anchor),
82 + accession_number=accession_number, cik=cik,
83 + section_id=section_id, section_name=section_name,
84 + project_name=pname, project_type=ptype,
85 + description=sent[:500],
86 + amount_usd=amount, amount_raw=amount_raw,
87 + locations=locations, technologies=techs, years=years,
88 + status=status, confidence=round(conf, 3), backend=self.name,
89 + ))
90 + return out
91 +
92 + @staticmethod
93 + def _synth_name(ptype: str, locations: list[str], techs: list[str]) -> str:
94 + label = tx.TYPE_LABELS.get(ptype, ptype)
95 + if locations:
96 + return f"{label} — {locations[0]}"
97 + if techs:
98 + return f"{label} ({techs[0]})"
99 + return label
100 +
101 + @staticmethod
102 + def _confidence(ptype, amount, locations, years, status, nm, low) -> float:
103 + c = 0.40
104 + if nm:
105 + c += 0.20
106 + if amount:
107 + c += 0.18
108 + if locations:
109 + c += 0.10
110 + if years:
111 + c += 0.06
112 + if status in ("planned", "in_progress", "completed"):
113 + c += 0.08
114 + # strong, specific phrasings
115 + if re.search(r"\b(announced|committed|invest\w+|will (build|open|invest))\b", low):
116 + c += 0.06
117 + # de-weight vague risk-factor / forward-looking hedges
118 + if re.search(r"\b(may|might|could|potential\w*|if we|uncertain)\b", low):
119 + c -= 0.08
120 + return max(0.0, min(1.0, c))
added code/audit/spid_snapshot/store.py +239 −0
@@ -0,0 +1,239 @@
1 +# ============================================================
2 +# SPID — DuckDB store
3 +#
4 +# Source corpus (filings / sections / companies) is exposed as
5 +# read-only views over the existing parquet. SPID writes its
6 +# own tables: mentions, projects, timeline, graph, embeddings.
7 +# ============================================================
8 +from __future__ import annotations
9 +
10 +from pathlib import Path
11 +
12 +import duckdb
13 +
14 +from spid.config import Settings, get_settings
15 +from spid.extract.base import ProjectMention
16 +
17 +SCHEMA = """
18 +CREATE TABLE IF NOT EXISTS project_mentions (
19 + mention_id VARCHAR PRIMARY KEY,
20 + accession_number VARCHAR,
21 + cik VARCHAR,
22 + section_id VARCHAR,
23 + section_name VARCHAR,
24 + project_name VARCHAR,
25 + project_type VARCHAR,
26 + description VARCHAR,
27 + objective VARCHAR,
28 + amount_usd DOUBLE,
29 + amount_raw VARCHAR,
30 + locations VARCHAR[],
31 + technologies VARCHAR[],
32 + partners VARCHAR[],
33 + suppliers VARCHAR[],
34 + benefits VARCHAR[],
35 + risks VARCHAR[],
36 + years INTEGER[],
37 + status VARCHAR,
38 + confidence DOUBLE,
39 + backend VARCHAR,
40 + -- denormalised filing context (filled on load)
41 + ticker VARCHAR,
42 + company_name VARCHAR,
43 + form_type VARCHAR,
44 + filing_date DATE,
45 + fiscal_year INTEGER,
46 + sector VARCHAR
47 +);
48 +
49 +CREATE TABLE IF NOT EXISTS projects (
50 + project_id VARCHAR PRIMARY KEY,
51 + cik VARCHAR,
52 + ticker VARCHAR,
53 + company_name VARCHAR,
54 + sector VARCHAR,
55 + project_type VARCHAR,
56 + project_name VARCHAR,
57 + canonical_location VARCHAR,
58 + technologies VARCHAR[],
59 + status VARCHAR,
60 + total_amount_usd DOUBLE,
61 + n_mentions INTEGER,
62 + n_filings INTEGER,
63 + first_seen DATE,
64 + last_seen DATE,
65 + first_year INTEGER,
66 + last_year INTEGER,
67 + avg_confidence DOUBLE,
68 + description VARCHAR
69 +);
70 +
71 +CREATE TABLE IF NOT EXISTS project_timeline (
72 + project_id VARCHAR,
73 + accession_number VARCHAR,
74 + filing_date DATE,
75 + form_type VARCHAR,
76 + status VARCHAR,
77 + amount_usd DOUBLE,
78 + confidence DOUBLE,
79 + snippet VARCHAR
80 +);
81 +
82 +CREATE TABLE IF NOT EXISTS kg_nodes (
83 + node_id VARCHAR PRIMARY KEY,
84 + node_type VARCHAR, -- Company | Project | Technology | Location | Partner
85 + label VARCHAR,
86 + props JSON
87 +);
88 +
89 +CREATE TABLE IF NOT EXISTS kg_edges (
90 + src VARCHAR,
91 + rel VARCHAR, -- owns | uses | located_in | partners_with | requires_capital
92 + dst VARCHAR,
93 + props JSON
94 +);
95 +
96 +CREATE TABLE IF NOT EXISTS project_embeddings (
97 + project_id VARCHAR PRIMARY KEY,
98 + model VARCHAR,
99 + dim INTEGER,
100 + vector FLOAT[]
101 +);
102 +"""
103 +
104 +
105 +class Store:
106 + def __init__(self, settings: Settings | None = None, read_only: bool = False):
107 + self.s = settings or get_settings()
108 + self.s.db_path.parent.mkdir(parents=True, exist_ok=True)
109 + self.con = duckdb.connect(str(self.s.db_path), read_only=read_only)
110 + if not read_only:
111 + self._init()
112 +
113 + def _init(self) -> None:
114 + self.con.execute(
115 + "CREATE TABLE IF NOT EXISTS spid_sections ("
116 + "section_id VARCHAR PRIMARY KEY, accession_number VARCHAR, section_name VARCHAR,"
117 + "item_number VARCHAR, text VARCHAR, word_count INTEGER)")
118 + self._register_sources()
119 + self.con.execute(SCHEMA)
120 +
121 + def _register_sources(self) -> None:
122 + c = self.con
123 + c.execute(
124 + "CREATE TABLE IF NOT EXISTS extracted_sections ("
125 + "section_id VARCHAR PRIMARY KEY, accession_number VARCHAR, backend VARCHAR,"
126 + "n_mentions INTEGER, extracted_at TIMESTAMP DEFAULT current_timestamp)")
127 + c.execute(f"CREATE OR REPLACE VIEW src_filings AS SELECT * FROM read_parquet('{self.s.filings_parquet}')")
128 + c.execute(f"CREATE OR REPLACE VIEW src_sections AS SELECT * FROM read_parquet('{self.s.sections_parquet}')")
129 + c.execute(f"CREATE OR REPLACE VIEW src_companies AS SELECT * FROM read_parquet('{self.s.companies_parquet}')")
130 + # unified, enriched section feed: pre-parsed sections (all forms since 2026-10-27)
131 + # UNION ingested 10-K items (legacy S&P 500 run); a 10-K already ingested in
132 + # spid_sections is not read a second time from the parsed corpus.
133 + scope = self.s.section_scope
134 + scope_sql = ""
135 + if scope:
136 + parts = []
137 + for form, items in scope.items():
138 + q = ",".join("'" + i + "'" for i in items)
139 + parts.append(f"(form_type = '{form}' AND section_name IN ({q}))")
140 + others = ",".join("'" + f + "'" for f in scope)
141 + scope_sql = " WHERE " + " OR ".join(parts) + f" OR form_type NOT IN ({others})"
142 + c.execute(f"""
143 + CREATE OR REPLACE VIEW v_sections AS
144 + WITH parsed AS (
145 + SELECT s.section_id, s.accession_number, s.section_name, s.item_number, s.text, s.word_count,
146 + f.form_type
147 + FROM src_sections s JOIN src_filings f USING (accession_number)
148 + WHERE NOT (f.form_type = '10-K'
149 + AND s.accession_number IN (SELECT DISTINCT accession_number FROM spid_sections))
150 + ),
151 + all_sec AS (
152 + SELECT section_id, accession_number, section_name, item_number, text, word_count
153 + FROM parsed{scope_sql}
154 + UNION ALL
155 + SELECT section_id, accession_number, section_name, item_number, text, word_count
156 + FROM spid_sections
157 + )
158 + SELECT s.section_id, s.accession_number, s.section_name, s.item_number,
159 + s.text, s.word_count,
160 + f.cik, f.ticker, f.company_name AS filing_company, f.form_type,
161 + CAST(f.filing_date AS DATE) AS filing_date, f.fiscal_year,
162 + co.company_name AS company_name, co.sector, co.sub_industry, co.sic_code, co.state
163 + FROM all_sec s
164 + JOIN src_filings f USING (accession_number)
165 + LEFT JOIN src_companies co ON f.cik = co.cik
166 + """)
167 +
168 + # ---- read side ----------------------------------------------------
169 + def companies(self) -> list[tuple]:
170 + return self.con.execute(
171 + "SELECT DISTINCT cik, ticker FROM src_filings ORDER BY cik"
172 + ).fetchall()
173 +
174 + def sections_for(self, *, ciks: list[str] | None = None, forms: list[str] | None = None,
175 + limit: int | None = None, min_words: int = 30, skip_done: bool = True) -> list[dict]:
176 + where = [f"word_count >= {int(min_words)}"]
177 + if skip_done:
178 + where.append("section_id NOT IN (SELECT section_id FROM extracted_sections)")
179 + if ciks:
180 + quoted = ",".join("'" + c.replace("'", "") + "'" for c in ciks)
181 + where.append(f"cik IN ({quoted})")
182 + if forms:
183 + quoted = ",".join("'" + f.replace("'", "") + "'" for f in forms)
184 + where.append(f"form_type IN ({quoted})")
185 + sql = "SELECT * FROM v_sections WHERE " + " AND ".join(where)
186 + sql += " ORDER BY filing_date"
187 + if limit:
188 + sql += f" LIMIT {int(limit)}"
189 + res = self.con.execute(sql)
190 + table = res.to_arrow_table() if hasattr(res, "to_arrow_table") else res.fetch_arrow_table() # DuckDB >= 2.0 / < 2.0
191 + return table.to_pylist()
192 +
193 + # ---- write side ---------------------------------------------------
194 + def upsert_mentions(self, mentions: list[ProjectMention], ctx: dict) -> int:
195 + """ctx maps accession_number -> filing context dict."""
196 + if not mentions:
197 + return 0
198 + rows = []
199 + for m in mentions:
200 + cx = ctx.get(m.accession_number, {})
201 + rows.append((
202 + m.mention_id, m.accession_number, m.cik, m.section_id, m.section_name,
203 + m.project_name, m.project_type, m.description, m.objective,
204 + m.amount_usd, m.amount_raw, m.locations, m.technologies, m.partners,
205 + m.suppliers, m.benefits, m.risks, m.years, m.status, m.confidence, m.backend,
206 + cx.get("ticker"), cx.get("company_name") or cx.get("filing_company"),
207 + cx.get("form_type"), cx.get("filing_date"), cx.get("fiscal_year"), cx.get("sector"),
208 + ))
209 + self.con.executemany(
210 + "INSERT OR REPLACE INTO project_mentions VALUES "
211 + "(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", rows)
212 + return len(rows)
213 +
214 + def mark_extracted(self, rows: list[tuple]) -> None:
215 + """rows: (section_id, accession_number, backend, n_mentions)."""
216 + if rows:
217 + self.con.executemany(
218 + "INSERT OR REPLACE INTO extracted_sections (section_id, accession_number, backend, n_mentions) "
219 + "VALUES (?,?,?,?)", rows)
220 +
221 + def backfill_extracted(self, backend: str = "llm") -> int:
222 + """Record every section currently visible in v_sections as already extracted.
223 +
224 + Used once, before the Russell 3000 corpus is merged, so the legacy S&P 500 run
225 + (whose sections were processed without a ledger) is not re-billed.
226 + """
227 + self.con.execute(f"""
228 + INSERT OR REPLACE INTO extracted_sections (section_id, accession_number, backend, n_mentions)
229 + SELECT v.section_id, v.accession_number, '{backend}',
230 + (SELECT COUNT(*) FROM project_mentions m WHERE m.section_id = v.section_id)
231 + FROM v_sections v
232 + """)
233 + return self.count("extracted_sections")
234 +
235 + def count(self, table: str) -> int:
236 + return self.con.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]
237 +
238 + def close(self) -> None:
239 + self.con.close()
added code/audit/spid_snapshot/taxonomy.py +261 −0
@@ -0,0 +1,261 @@
1 +# ============================================================
2 +# SPID — project taxonomy & lightweight value extractors
3 +#
4 +# Shared by both the rule-based and the LLM extractor so that
5 +# `project_type`, `technology`, money, location and dates are
6 +# normalised the same way regardless of backend.
7 +# ============================================================
8 +from __future__ import annotations
9 +
10 +import re
11 +
12 +# ----------------------------------------------------------------------
13 +# Canonical project types. `signals` are high-precision regex fragments
14 +# (already lower-cased) used by the rule extractor and as the controlled
15 +# vocabulary the LLM is asked to classify into.
16 +# ----------------------------------------------------------------------
17 +PROJECT_TYPES: dict[str, dict] = {
18 + "manufacturing_expansion": {
19 + "label": "Manufacturing / Capacity Expansion",
20 + "signals": [
21 + r"(expand|increas\w+|add\w*|ramp\w*|scal\w+|new|build\w*|construct\w*|open\w*)"
22 + r"[^.]{0,60}(manufactur\w+|production|plant|factory|facilit\w+|capacity|assembly|fab\b)",
23 + ],
24 + },
25 + "plant_construction": {
26 + "label": "Plant / Site Construction",
27 + "signals": [r"(construct\w*|build\w*|broke ground|commission\w*|new (plant|site|facility))"
28 + r"[^.]{0,60}(plant|facility|site|campus|complex|refinery|mill)"],
29 + },
30 + "data_center": {
31 + "label": "Data Center",
32 + "signals": [r"data\s?cent(er|re)s?", r"hyperscal\w+", r"server farm"],
33 + },
34 + "cloud_migration": {
35 + "label": "Cloud Migration",
36 + "signals": [r"(migrat\w+|mov\w+|transition\w*)[^.]{0,40}(cloud)",
37 + r"cloud (migration|platform|adoption|infrastructure|first)"],
38 + },
39 + "ai_initiative": {
40 + "label": "AI / ML Initiative",
41 + "signals": [r"\b(artificial intelligence|machine learning|generative ai|gen ?ai|deep learning|"
42 + r"large language model|\bllm\b|neural network)\b",
43 + r"\bai[- ](powered|driven|enabled|capabilit\w+|platform|model|tools?)\b"],
44 + },
45 + "digital_transformation": {
46 + "label": "Digital Transformation",
47 + "signals": [r"digital (transformation|transition|initiative|strateg\w+|platform)",
48 + r"digitali[sz]ation"],
49 + },
50 + "erp_implementation": {
51 + "label": "ERP / Systems Implementation",
52 + "signals": [r"\b(erp|sap|oracle|workday|s/?4 ?hana)\b[^.]{0,40}(implement\w+|deploy\w+|migrat\w+|system|platform)",
53 + r"(implement\w+|deploy\w+|upgrad\w+)[^.]{0,40}\berp\b",
54 + r"enterprise resource planning"],
55 + },
56 + "rd_program": {
57 + "label": "R&D Program",
58 + "signals": [r"research and development", r"\br&d\b",
59 + r"(develop\w+|invest\w+)[^.]{0,40}(next[- ]generation|new technolog\w+|new product)"],
60 + },
61 + "product_launch": {
62 + "label": "New Product / Launch",
63 + "signals": [r"(launch\w*|introduc\w+|releas\w+|roll\w*[- ]?out|unveil\w+)"
64 + r"[^.]{0,50}(product|platform|service|offering|solution|model|line)",
65 + r"new product (line|launch|introduction)"],
66 + },
67 + "drug_pipeline": {
68 + "label": "Drug / Clinical Pipeline",
69 + "signals": [r"(phase\s?(i{1,3}|1|2|3)\b)", r"clinical trial", r"\bfda (approval|clearance|submission)\b",
70 + r"(drug|therap\w+|candidate|molecule|pipeline)[^.]{0,40}(develop\w+|trial|approval)"],
71 + },
72 + "ma_integration": {
73 + "label": "M&A / Integration",
74 + "signals": [r"(acqui\w+|merg\w+|takeover|business combination)",
75 + r"integration of [a-z]", r"purchase of [a-z]"],
76 + },
77 + "geographic_expansion": {
78 + "label": "Geographic Expansion",
79 + "signals": [r"(expand\w*|enter\w*|launch\w*|establish\w*)[^.]{0,50}"
80 + r"(market|region|countr\w+|geograph\w+|international|overseas|abroad)",
81 + r"market entry"],
82 + },
83 + "capex_program": {
84 + "label": "Capital Expenditure Program",
85 + "signals": [r"capital (expenditure|expenditures|investment|spending|program|project|budget)",
86 + r"\bcapex\b", r"invest\w+[^.]{0,30}(\$|usd)"],
87 + },
88 + "supply_chain": {
89 + "label": "Supply Chain Initiative",
90 + "signals": [r"supply chain", r"(reshor\w+|nearshor\w+|onshor\w+|offshor\w+)",
91 + r"(distribution|logistics|fulfillment|warehouse|inventory)[^.]{0,40}"
92 + r"(network|center|optimi[sz]\w+|expansion|investment)"],
93 + },
94 + "automation": {
95 + "label": "Automation / Robotics",
96 + "signals": [r"automat\w+", r"robot\w+", r"(industrial|warehouse) automation"],
97 + },
98 + "sustainability": {
99 + "label": "Sustainability / ESG",
100 + "signals": [r"sustainabilit\w+", r"\besg\b", r"net[- ]zero", r"carbon (neutral|reduction|footprint|emissions?)",
101 + r"decarboni[sz]\w+", r"emission reduction"],
102 + },
103 + "energy_transition": {
104 + "label": "Energy Transition / Renewables",
105 + "signals": [r"renewable (energy|power)", r"(solar|wind|hydrogen|geothermal|battery storage)[^.]{0,40}"
106 + r"(project|facility|farm|plant|capacity|investment)",
107 + r"clean energy", r"energy transition"],
108 + },
109 + "infrastructure": {
110 + "label": "Infrastructure Investment",
111 + "signals": [r"infrastructure (investment|project|upgrade|expansion|spending|build\w*)",
112 + r"(network|grid|pipeline|broadband|fiber) (expansion|build\w*|upgrade|deployment)"],
113 + },
114 + "cost_reduction": {
115 + "label": "Cost Reduction / Restructuring",
116 + "signals": [r"(cost (reduction|savings|cutting)|restructur\w+|efficiency (program|initiative)|"
117 + r"productivity (program|initiative)|streamlin\w+|optimi[sz]ation (program|initiative))"],
118 + },
119 + "partnership": {
120 + "label": "Strategic Partnership",
121 + "signals": [r"(strategic (partnership|alliance|collaboration)|joint venture|\bjv\b|"
122 + r"(partner\w+|collaborat\w+|team\w* up)[^.]{0,40}(with))"],
123 + },
124 + "technology_deployment": {
125 + "label": "Technology Deployment",
126 + "signals": [r"(deploy\w+|implement\w+|roll\w*[- ]?out|adopt\w+|install\w+)[^.]{0,50}"
127 + r"(technolog\w+|system|platform|software|solution|tool)"],
128 + },
129 +}
130 +
131 +# ----------------------------------------------------------------------
132 +# Technology vocabulary -> canonical tag
133 +# ----------------------------------------------------------------------
134 +TECHNOLOGY_TERMS: dict[str, str] = {
135 + r"\b(artificial intelligence|machine learning|generative ai|gen ?ai|deep learning|llm|neural)\b": "AI/ML",
136 + r"\bcloud\b": "Cloud",
137 + r"\b(erp|sap|oracle|workday|s/?4 ?hana|peoplesoft)\b": "ERP",
138 + r"\b(robot\w+|automation|cobots?)\b": "Robotics/Automation",
139 + r"\b(semiconductor|chip|wafer|fab|node\b|nanometer|foundry)\b": "Semiconductors",
140 + r"\b(solar|wind|renewable|hydrogen|geothermal)\b": "Renewable Energy",
141 + r"\b(battery|batteries|energy storage|lithium)\b": "Battery/Storage",
142 + r"\b(ev|electric vehicle|charging)\b": "Electric Vehicles",
143 + r"\b(blockchain|crypto|digital asset|tokeni[sz])\b": "Blockchain",
144 + r"\b(5g|broadband|fiber|fibre|network infrastructure)\b": "Connectivity",
145 + r"\b(data\s?cent(er|re)|hyperscale|server)\b": "Data Center",
146 + r"\b(cyber ?security|encryption|zero trust)\b": "Cybersecurity",
147 + r"\b(iot|internet of things|sensors?|edge computing)\b": "IoT/Edge",
148 + r"\b(quantum)\b": "Quantum",
149 + r"\b(biolog\w+|gene|mrna|crispr|cell therapy|molecul\w+)\b": "Biotech",
150 + r"\b(saas|software[- ]as[- ]a[- ]service|platform)\b": "Software/SaaS",
151 +}
152 +
153 +# ----------------------------------------------------------------------
154 +# Status keywords -> canonical status
155 +# ----------------------------------------------------------------------
156 +STATUS_TERMS: list[tuple[str, str]] = [
157 + (r"\b(completed|commission\w+|operational|in service|came online|opened|finalized)\b", "completed"),
158 + (r"\b(under construction|underway|in progress|ongoing|ramping|being (built|implemented))\b", "in_progress"),
159 + (r"\b(announc\w+|plan\w+|intend\w+|expect\w+|will (build|invest|launch|open)|propos\w+|"
160 + r"anticipat\w+|target\w+|by (20\d\d)|slated)\b", "planned"),
161 +]
162 +
163 +# ----------------------------------------------------------------------
164 +# Geography
165 +# ----------------------------------------------------------------------
166 +US_STATES = {
167 + "alabama","alaska","arizona","arkansas","california","colorado","connecticut","delaware",
168 + "florida","georgia","hawaii","idaho","illinois","indiana","iowa","kansas","kentucky",
169 + "louisiana","maine","maryland","massachusetts","michigan","minnesota","mississippi",
170 + "missouri","montana","nebraska","nevada","new hampshire","new jersey","new mexico",
171 + "new york","north carolina","north dakota","ohio","oklahoma","oregon","pennsylvania",
172 + "rhode island","south carolina","south dakota","tennessee","texas","utah","vermont",
173 + "virginia","washington","west virginia","wisconsin","wyoming",
174 +}
175 +COUNTRIES = {
176 + "united states","u.s.","usa","canada","mexico","china","india","japan","germany","france",
177 + "united kingdom","u.k.","ireland","brazil","singapore","south korea","korea","taiwan",
178 + "vietnam","thailand","malaysia","indonesia","australia","spain","italy","netherlands",
179 + "switzerland","sweden","poland","israel","saudi arabia","united arab emirates","uae",
180 + "philippines","czech republic","hungary","turkey","egypt","south africa","argentina","chile",
181 +}
182 +
183 +_MONEY_RE = re.compile(
184 + r"(?:US)?\$\s?(\d[\d,]*\.?\d*)\s?(billion|bn|million|mm|mn|m\b|thousand|k\b|trillion)?",
185 + re.IGNORECASE,
186 +)
187 +_MULT = {
188 + "trillion": 1e12, "billion": 1e9, "bn": 1e9, "million": 1e6, "mm": 1e6, "mn": 1e6,
189 + "m": 1e6, "thousand": 1e3, "k": 1e3,
190 +}
191 +_YEAR_RE = re.compile(r"\b(?:fiscal\s+|fy\s?)?(20[0-3]\d|19[89]\d)\b", re.IGNORECASE)
192 +_QUARTER_RE = re.compile(r"\bQ([1-4])\s?(20\d\d)\b", re.IGNORECASE)
193 +_TECH_COMPILED = [(re.compile(p, re.IGNORECASE), tag) for p, tag in TECHNOLOGY_TERMS.items()]
194 +_STATUS_COMPILED = [(re.compile(p, re.IGNORECASE), s) for p, s in STATUS_TERMS]
195 +_CITY_STATE_RE = re.compile(r"\bin\s+([A-Z][a-zA-Z.]+(?:\s[A-Z][a-zA-Z.]+){0,2}),\s+([A-Z][a-zA-Z]+)")
196 +
197 +
198 +def extract_money(text: str) -> tuple[float | None, str | None]:
199 + """Return (usd_amount, raw_span) for the largest explicit dollar figure."""
200 + best_val, best_raw = None, None
201 + for m in _MONEY_RE.finditer(text):
202 + try:
203 + num = float(m.group(1).replace(",", ""))
204 + except ValueError:
205 + continue
206 + unit = (m.group(2) or "").lower().strip()
207 + val = num * _MULT.get(unit, 1.0)
208 + if best_val is None or val > best_val:
209 + best_val, best_raw = val, m.group(0).strip()
210 + return best_val, best_raw
211 +
212 +
213 +def extract_locations(text: str) -> list[str]:
214 + low = text.lower()
215 + found: list[str] = []
216 + for st in US_STATES:
217 + if re.search(rf"\b{re.escape(st)}\b", low):
218 + found.append(st.title())
219 + for co in COUNTRIES:
220 + if re.search(rf"\b{re.escape(co)}\b", low):
221 + found.append(co.upper() if len(co) <= 4 else co.title())
222 + for m in _CITY_STATE_RE.finditer(text):
223 + found.append(f"{m.group(1)}, {m.group(2)}")
224 + # de-dup preserving order
225 + seen, out = set(), []
226 + for f in found:
227 + k = f.lower()
228 + if k not in seen:
229 + seen.add(k); out.append(f)
230 + return out[:6]
231 +
232 +
233 +def extract_years(text: str) -> list[int]:
234 + yrs = {int(y) for y in _YEAR_RE.findall(text)}
235 + for m in _QUARTER_RE.finditer(text):
236 + yrs.add(int(m.group(2)))
237 + return sorted(yrs)
238 +
239 +
240 +def extract_technologies(text: str) -> list[str]:
241 + tags = []
242 + for rx, tag in _TECH_COMPILED:
243 + if rx.search(text) and tag not in tags:
244 + tags.append(tag)
245 + return tags
246 +
247 +
248 +def detect_status(text: str) -> str:
249 + for rx, status in _STATUS_COMPILED:
250 + if rx.search(text):
251 + return status
252 + return "mentioned"
253 +
254 +
255 +# pre-compile project-type signals
256 +TYPE_SIGNALS = {
257 + ptype: [re.compile(sig, re.IGNORECASE) for sig in spec["signals"]]
258 + for ptype, spec in PROJECT_TYPES.items()
259 +}
260 +TYPE_LABELS = {k: v["label"] for k, v in PROJECT_TYPES.items()}
261 +ALL_TYPES = list(PROJECT_TYPES.keys())
added results/n0_probe/manifest.json +27 −0
@@ -0,0 +1,27 @@
1 +{
2 + "package": "n0_probe",
3 + "script": "/Users/simon-pierreboucher/firc0001_review/scripts/a00_probe_schema.py",
4 + "script_sha256": "b3bd5ca2a11782452a7d9c78b7f1ff63236fc60847ad4fc16914c1383f3b5a93",
5 + "started": "2026-11-05T14:10:08",
6 + "ended": "2026-11-05T14:10:17",
7 + "wall_seconds": 8.3,
8 + "node": "m3u96b",
9 + "python": "3.12.15",
10 + "duckdb": "2.0.0",
11 + "threads": 4,
12 + "memory_limit": "24GB",
13 + "seed": 20261105,
14 + "databases": {
15 + "/Users/simon-pierreboucher/UQO_RESEARCH_DB/project-db/data/db/spid.duckdb": {
16 + "bytes": 3155701760,
17 + "mtime": "2026-10-28T12:21:11.091389"
18 + },
19 + "/Users/simon-pierreboucher/UQO_RESEARCH_DB/data/duckdb/edgar_research.duckdb": {
20 + "bytes": 73500995584,
21 + "mtime": "2026-10-27T21:34:46.297884"
22 + }
23 + },
24 + "read_only": true,
25 + "row_counts": {},
26 + "notes": {}
27 +}
\ No newline at end of file
added results/n0_probe/schema.json +937 −0
@@ -0,0 +1,937 @@
1 +{
2 + "extracted_sections": [
3 + [
4 + "section_id",
5 + "VARCHAR"
6 + ],
7 + [
8 + "accession_number",
9 + "VARCHAR"
10 + ],
11 + [
12 + "backend",
13 + "VARCHAR"
14 + ],
15 + [
16 + "n_mentions",
17 + "INTEGER"
18 + ],
19 + [
20 + "extracted_at",
21 + "TIMESTAMP"
22 + ]
23 + ],
24 + "project_mentions": [
25 + [
26 + "mention_id",
27 + "VARCHAR"
28 + ],
29 + [
30 + "accession_number",
31 + "VARCHAR"
32 + ],
33 + [
34 + "cik",
35 + "VARCHAR"
36 + ],
37 + [
38 + "section_id",
39 + "VARCHAR"
40 + ],
41 + [
42 + "section_name",
43 + "VARCHAR"
44 + ],
45 + [
46 + "project_name",
47 + "VARCHAR"
48 + ],
49 + [
50 + "project_type",
51 + "VARCHAR"
52 + ],
53 + [
54 + "description",
55 + "VARCHAR"
56 + ],
57 + [
58 + "objective",
59 + "VARCHAR"
60 + ],
61 + [
62 + "amount_usd",
63 + "DOUBLE"
64 + ],
65 + [
66 + "amount_raw",
67 + "VARCHAR"
68 + ],
69 + [
70 + "locations",
71 + "VARCHAR[]"
72 + ],
73 + [
74 + "technologies",
75 + "VARCHAR[]"
76 + ],
77 + [
78 + "partners",
79 + "VARCHAR[]"
80 + ],
81 + [
82 + "suppliers",
83 + "VARCHAR[]"
84 + ],
85 + [
86 + "benefits",
87 + "VARCHAR[]"
88 + ],
89 + [
90 + "risks",
91 + "VARCHAR[]"
92 + ],
93 + [
94 + "years",
95 + "INTEGER[]"
96 + ],
97 + [
98 + "status",
99 + "VARCHAR"
100 + ],
101 + [
102 + "confidence",
103 + "DOUBLE"
104 + ],
105 + [
106 + "backend",
107 + "VARCHAR"
108 + ],
109 + [
110 + "ticker",
111 + "VARCHAR"
112 + ],
113 + [
114 + "company_name",
115 + "VARCHAR"
116 + ],
117 + [
118 + "form_type",
119 + "VARCHAR"
120 + ],
121 + [
122 + "filing_date",
123 + "DATE"
124 + ],
125 + [
126 + "fiscal_year",
127 + "INTEGER"
128 + ],
129 + [
130 + "sector",
131 + "VARCHAR"
132 + ]
133 + ],
134 + "projects": [
135 + [
136 + "project_id",
137 + "VARCHAR"
138 + ],
139 + [
140 + "cik",
141 + "VARCHAR"
142 + ],
143 + [
144 + "ticker",
145 + "VARCHAR"
146 + ],
147 + [
148 + "company_name",
149 + "VARCHAR"
150 + ],
151 + [
152 + "sector",
153 + "VARCHAR"
154 + ],
155 + [
156 + "project_type",
157 + "VARCHAR"
158 + ],
159 + [
160 + "project_name",
161 + "VARCHAR"
162 + ],
163 + [
164 + "canonical_location",
165 + "VARCHAR"
166 + ],
167 + [
168 + "technologies",
169 + "VARCHAR[]"
170 + ],
171 + [
172 + "status",
173 + "VARCHAR"
174 + ],
175 + [
176 + "total_amount_usd",
177 + "DOUBLE"
178 + ],
179 + [
180 + "n_mentions",
181 + "INTEGER"
182 + ],
183 + [
184 + "n_filings",
185 + "INTEGER"
186 + ],
187 + [
188 + "first_seen",
189 + "DATE"
190 + ],
191 + [
192 + "last_seen",
193 + "DATE"
194 + ],
195 + [
196 + "first_year",
197 + "INTEGER"
198 + ],
199 + [
200 + "last_year",
201 + "INTEGER"
202 + ],
203 + [
204 + "avg_confidence",
205 + "DOUBLE"
206 + ],
207 + [
208 + "description",
209 + "VARCHAR"
210 + ]
211 + ],
212 + "project_timeline": [
213 + [
214 + "project_id",
215 + "VARCHAR"
216 + ],
217 + [
218 + "accession_number",
219 + "VARCHAR"
220 + ],
221 + [
222 + "filing_date",
223 + "DATE"
224 + ],
225 + [
226 + "form_type",
227 + "VARCHAR"
228 + ],
229 + [
230 + "status",
231 + "VARCHAR"
232 + ],
233 + [
234 + "amount_usd",
235 + "DOUBLE"
236 + ],
237 + [
238 + "confidence",
239 + "DOUBLE"
240 + ],
241 + [
242 + "snippet",
243 + "VARCHAR"
244 + ]
245 + ],
246 + "spid_sections": [
247 + [
248 + "section_id",
249 + "VARCHAR"
250 + ],
251 + [
252 + "accession_number",
253 + "VARCHAR"
254 + ],
255 + [
256 + "section_name",
257 + "VARCHAR"
258 + ],
259 + [
260 + "item_number",
261 + "VARCHAR"
262 + ],
263 + [
264 + "text",
265 + "VARCHAR"
266 + ],
267 + [
268 + "word_count",
269 + "INTEGER"
270 + ]
271 + ],
272 + "v_sections": [
273 + [
274 + "section_id",
275 + "VARCHAR"
276 + ],
277 + [
278 + "accession_number",
279 + "VARCHAR"
280 + ],
281 + [
282 + "section_name",
283 + "VARCHAR"
284 + ],
285 + [
286 + "item_number",
287 + "VARCHAR"
288 + ],
289 + [
290 + "text",
291 + "VARCHAR"
292 + ],
293 + [
294 + "word_count",
295 + "INTEGER"
296 + ],
297 + [
298 + "cik",
299 + "VARCHAR"
300 + ],
301 + [
302 + "ticker",
303 + "VARCHAR"
304 + ],
305 + [
306 + "filing_company",
307 + "VARCHAR"
308 + ],
309 + [
310 + "form_type",
311 + "VARCHAR"
312 + ],
313 + [
314 + "filing_date",
315 + "DATE"
316 + ],
317 + [
318 + "fiscal_year",
319 + "INTEGER"
320 + ],
321 + [
322 + "company_name",
323 + "VARCHAR"
324 + ],
325 + [
326 + "sector",
327 + "VARCHAR"
328 + ],
329 + [
330 + "sub_industry",
331 + "VARCHAR"
332 + ],
333 + [
334 + "sic_code",
335 + "VARCHAR"
336 + ],
337 + [
338 + "state",
339 + "VARCHAR"
340 + ]
341 + ],
342 + "src_filings": [
343 + [
344 + "accession_number",
345 + "VARCHAR"
346 + ],
347 + [
348 + "cik",
349 + "VARCHAR"
350 + ],
351 + [
352 + "ticker",
353 + "VARCHAR"
354 + ],
355 + [
356 + "company_name",
357 + "VARCHAR"
358 + ],
359 + [
360 + "form_type",
361 + "VARCHAR"
362 + ],
363 + [
364 + "filing_date",
365 + "DATE"
366 + ],
367 + [
368 + "report_date",
369 + "DATE"
370 + ],
371 + [
372 + "fiscal_year",
373 + "INTEGER"
374 + ],
375 + [
376 + "fiscal_quarter",
377 + "INTEGER"
378 + ],
379 + [
380 + "sec_url",
381 + "VARCHAR"
382 + ],
383 + [
384 + "local_path",
385 + "VARCHAR"
386 + ],
387 + [
388 + "primary_document",
389 + "VARCHAR"
390 + ],
391 + [
392 + "file_size_bytes",
393 + "BIGINT"
394 + ],
395 + [
396 + "download_status",
397 + "VARCHAR"
398 + ],
399 + [
400 + "parse_status",
401 + "VARCHAR"
402 + ],
403 + [
404 + "created_at",
405 + "TIMESTAMP"
406 + ],
407 + [
408 + "updated_at",
409 + "TIMESTAMP"
410 + ]
411 + ],
412 + "src_sections": [
413 + [
414 + "section_id",
415 + "VARCHAR"
416 + ],
417 + [
418 + "accession_number",
419 + "VARCHAR"
420 + ],
421 + [
422 + "section_name",
423 + "VARCHAR"
424 + ],
425 + [
426 + "item_number",
427 + "VARCHAR"
428 + ],
429 + [
430 + "section_order",
431 + "INTEGER"
432 + ],
433 + [
434 + "text",
435 + "VARCHAR"
436 + ],
437 + [
438 + "word_count",
439 + "INTEGER"
440 + ],
441 + [
442 + "char_count",
443 + "INTEGER"
444 + ],
445 + [
446 + "created_at",
447 + "TIMESTAMP"
448 + ]
449 + ],
450 + "src_companies": [
451 + [
452 + "cik",
453 + "VARCHAR"
454 + ],
455 + [
456 + "ticker",
457 + "VARCHAR"
458 + ],
459 + [
460 + "company_name",
461 + "VARCHAR"
462 + ],
463 + [
464 + "sector",
465 + "VARCHAR"
466 + ],
467 + [
468 + "sub_industry",
469 + "VARCHAR"
470 + ],
471 + [
472 + "sic_code",
473 + "VARCHAR"
474 + ],
475 + [
476 + "state",
477 + "VARCHAR"
478 + ],
479 + [
480 + "date_added",
481 + "DATE"
482 + ],
483 + [
484 + "created_at",
485 + "TIMESTAMP"
486 + ],
487 + [
488 + "updated_at",
489 + "TIMESTAMP"
490 + ],
491 + [
492 + "in_sp500",
493 + "BOOLEAN"
494 + ],
495 + [
496 + "in_russell3000",
497 + "BOOLEAN"
498 + ],
499 + [
500 + "sic_description",
501 + "VARCHAR"
502 + ],
503 + [
504 + "state_of_incorporation",
505 + "VARCHAR"
506 + ],
507 + [
508 + "exchange",
509 + "VARCHAR"
510 + ],
511 + [
512 + "fiscal_year_end",
513 + "VARCHAR"
514 + ],
515 + [
516 + "russell_weight_pct",
517 + "DOUBLE"
518 + ],
519 + [
520 + "shard_node",
521 + "VARCHAR"
522 + ]
523 + ],
524 + "views": [
525 + [
526 + "src_companies",
527 + "CREATE VIEW src_companies AS SELECT * FROM read_parquet('/Users/simon-pierreboucher/UQO_RESEARCH_DB/data/parquet/companies/companies.parquet');"
528 + ],
529 + [
530 + "src_filings",
531 + "CREATE VIEW src_filings AS SELECT * FROM read_parquet('/Users/simon-pierreboucher/UQO_RESEARCH_DB/data/parquet/filings/filings.parquet');"
532 + ],
533 + [
534 + "src_sections",
535 + "CREATE VIEW src_sections AS SELECT * FROM read_parquet('/Users/simon-pierreboucher/UQO_RESEARCH_DB/data/parquet/filing_sections/filing_sections.parquet');"
536 + ],
537 + [
538 + "v_sections",
539 + "CREATE VIEW v_sections AS WITH parsed AS (SELECT s.section_id, s.accession_number, s.section_name, s.item_number, s.\"text\", s.word_count, f.form_type FROM src_sections AS s INNER JOIN src_filings AS f USING (accession_number) WHERE (NOT ((f.form_type = '10-K') AND (s.accession_number = ANY(SELECT DISTINCT accession_number FROM spid_sections))))), all_sec AS ((SELECT section_id, accession_number, section_name, item_number, \"text\", word_count FROM parsed WHERE (((form_type = '10-K') AND (section_name IN ('business', 'risk_factors', 'properties', 'md_and_a', 'quantitative_disclosures'))) OR (NOT (form_type IN ('10-K'))))) UNION ALL (SELECT section_id, accession_number, section_name, item_number, \"text\", word_count FROM spid_sections)) SELECT s.section_id, s.accession_number, s.section_name, s.item_number, s.\"text\", s.word_count, f.cik, f.ticker, f.company_name AS filing_company, f.form_type, CAST(f.filing_date AS DATE) AS filing_date, f.fiscal_year, co.company_name AS company_name, co.sector, co.sub_industry, co.sic_code, co.state FROM all_sec AS s INNER JOIN src_filings AS f USING (accession_number) LEFT JOIN src_companies AS co ON ((f.cik = co.cik));"
540 + ]
541 + ],
542 + "counts": {
543 + "extracted_sections": 1123098,
544 + "project_mentions": 308065,
545 + "projects": 121099,
546 + "project_timeline": 308065,
547 + "spid_sections": 27164
548 + },
549 + "ledger_words": [
550 + 1123098,
551 + 3605932059
552 + ],
553 + "parquet_files": [
554 + 1342283,
555 + 29.13085581
556 + ],
557 + "edgar_filings": [
558 + [
559 + "accession_number",
560 + "VARCHAR"
561 + ],
562 + [
563 + "cik",
564 + "VARCHAR"
565 + ],
566 + [
567 + "ticker",
568 + "VARCHAR"
569 + ],
570 + [
571 + "company_name",
572 + "VARCHAR"
573 + ],
574 + [
575 + "form_type",
576 + "VARCHAR"
577 + ],
578 + [
579 + "filing_date",
580 + "DATE"
581 + ],
582 + [
583 + "report_date",
584 + "DATE"
585 + ],
586 + [
587 + "fiscal_year",
588 + "INTEGER"
589 + ],
590 + [
591 + "fiscal_quarter",
592 + "INTEGER"
593 + ],
594 + [
595 + "sec_url",
596 + "VARCHAR"
597 + ],
598 + [
599 + "local_path",
600 + "VARCHAR"
601 + ],
602 + [
603 + "primary_document",
604 + "VARCHAR"
605 + ],
606 + [
607 + "file_size_bytes",
608 + "BIGINT"
609 + ],
610 + [
611 + "download_status",
612 + "VARCHAR"
613 + ],
614 + [
615 + "parse_status",
616 + "VARCHAR"
617 + ],
618 + [
619 + "created_at",
620 + "TIMESTAMP"
621 + ],
622 + [
623 + "updated_at",
624 + "TIMESTAMP"
625 + ]
626 + ],
627 + "edgar_filing_sections": [
628 + [
629 + "section_id",
630 + "VARCHAR"
631 + ],
632 + [
633 + "accession_number",
634 + "VARCHAR"
635 + ],
636 + [
637 + "section_name",
638 + "VARCHAR"
639 + ],
640 + [
641 + "item_number",
642 + "VARCHAR"
643 + ],
644 + [
645 + "section_order",
646 + "INTEGER"
647 + ],
648 + [
649 + "text",
650 + "VARCHAR"
651 + ],
652 + [
653 + "word_count",
654 + "INTEGER"
655 + ],
656 + [
657 + "char_count",
658 + "INTEGER"
659 + ],
660 + [
661 + "created_at",
662 + "TIMESTAMP"
663 + ]
664 + ],
665 + "edgar_companies": [
666 + [
667 + "cik",
668 + "VARCHAR"
669 + ],
670 + [
671 + "ticker",
672 + "VARCHAR"
673 + ],
674 + [
675 + "company_name",
676 + "VARCHAR"
677 + ],
678 + [
679 + "sector",
680 + "VARCHAR"
681 + ],
682 + [
683 + "sub_industry",
684 + "VARCHAR"
685 + ],
686 + [
687 + "sic_code",
688 + "VARCHAR"
689 + ],
690 + [
691 + "state",
692 + "VARCHAR"
693 + ],
694 + [
695 + "date_added",
696 + "DATE"
697 + ],
698 + [
699 + "created_at",
700 + "TIMESTAMP"
701 + ],
702 + [
703 + "updated_at",
704 + "TIMESTAMP"
705 + ],
706 + [
707 + "in_sp500",
708 + "BOOLEAN"
709 + ],
710 + [
711 + "in_russell3000",
712 + "BOOLEAN"
713 + ],
714 + [
715 + "sic_description",
716 + "VARCHAR"
717 + ],
718 + [
719 + "state_of_incorporation",
720 + "VARCHAR"
721 + ],
722 + [
723 + "exchange",
724 + "VARCHAR"
725 + ],
726 + [
727 + "fiscal_year_end",
728 + "VARCHAR"
729 + ],
730 + [
731 + "russell_weight_pct",
732 + "DOUBLE"
733 + ],
734 + [
735 + "shard_node",
736 + "VARCHAR"
737 + ]
738 + ],
739 + "edgar_annual_panel": [
740 + [
741 + "cik",
742 + "VARCHAR"
743 + ],
744 + [
745 + "ticker",
746 + "VARCHAR"
747 + ],
748 + [
749 + "fiscal_year",
750 + "INTEGER"
751 + ],
752 + [
753 + "filing_count",
754 + "BIGINT"
755 + ],
756 + [
757 + "total_chunks",
758 + "BIGINT"
759 + ],
760 + [
761 + "ai_adoption_mean",
762 + "DOUBLE"
763 + ],
764 + [
765 + "ai_adoption_max",
766 + "DOUBLE"
767 + ],
768 + [
769 + "ai_adoption_std",
770 + "DOUBLE"
771 + ],
772 + [
773 + "ai_adoption_top5pct",
774 + "DOUBLE"
775 + ],
776 + [
777 + "ai_adoption_wmean",
778 + "DOUBLE"
779 + ],
780 + [
781 + "ai_disruption_risk_mean",
782 + "DOUBLE"
783 + ],
784 + [
785 + "ai_disruption_risk_max",
786 + "DOUBLE"
787 + ],
788 + [
789 + "ai_disruption_risk_std",
790 + "DOUBLE"
791 + ],
792 + [
793 + "ai_disruption_risk_top5pct",
794 + "DOUBLE"
795 + ],
796 + [
797 + "ai_disruption_risk_wmean",
798 + "DOUBLE"
799 + ],
800 + [
801 + "ai_infrastructure_mean",
802 + "DOUBLE"
803 + ],
804 + [
805 + "ai_infrastructure_max",
806 + "DOUBLE"
807 + ],
808 + [
809 + "ai_infrastructure_std",
810 + "DOUBLE"
811 + ],
812 + [
813 + "ai_infrastructure_top5pct",
814 + "DOUBLE"
815 + ],
816 + [
817 + "ai_infrastructure_wmean",
818 + "DOUBLE"
819 + ],
820 + [
821 + "esg_commitment_mean",
822 + "DOUBLE"
823 + ],
824 + [
825 + "esg_commitment_max",
826 + "DOUBLE"
827 + ],
828 + [
829 + "esg_commitment_std",
830 + "DOUBLE"
831 + ],
832 + [
833 + "esg_commitment_top5pct",
834 + "DOUBLE"
835 + ],
836 + [
837 + "esg_commitment_wmean",
838 + "DOUBLE"
839 + ],
840 + [
841 + "climate_risk_mean",
842 + "DOUBLE"
843 + ],
844 + [
845 + "climate_risk_max",
846 + "DOUBLE"
847 + ],
848 + [
849 + "climate_risk_std",
850 + "DOUBLE"
851 + ],
852 + [
853 + "climate_risk_top5pct",
854 + "DOUBLE"
855 + ],
856 + [
857 + "climate_risk_wmean",
858 + "DOUBLE"
859 + ],
860 + [
861 + "cybersecurity_mean",
862 + "DOUBLE"
863 + ],
864 + [
865 + "cybersecurity_max",
866 + "DOUBLE"
867 + ],
868 + [
869 + "cybersecurity_std",
870 + "DOUBLE"
871 + ],
872 + [
873 + "cybersecurity_top5pct",
874 + "DOUBLE"
875 + ],
876 + [
877 + "cybersecurity_wmean",
878 + "DOUBLE"
879 + ],
880 + [
881 + "automation_labor_mean",
882 + "DOUBLE"
883 + ],
884 + [
885 + "automation_labor_max",
886 + "DOUBLE"
887 + ],
888 + [
889 + "automation_labor_std",
890 + "DOUBLE"
891 + ],
892 + [
893 + "automation_labor_top5pct",
894 + "DOUBLE"
895 + ],
896 + [
897 + "automation_labor_wmean",
898 + "DOUBLE"
899 + ],
900 + [
901 + "macro_risk_mean",
902 + "DOUBLE"
903 + ],
904 + [
905 + "macro_risk_max",
906 + "DOUBLE"
907 + ],
908 + [
909 + "macro_risk_std",
910 + "DOUBLE"
911 + ],
912 + [
913 + "macro_risk_top5pct",
914 + "DOUBLE"
915 + ],
916 + [
917 + "macro_risk_wmean",
918 + "DOUBLE"
919 + ]
920 + ],
921 + "edgar_tables": [
922 + "anchor_embeddings",
923 + "annual_final",
924 + "annual_panel",
925 + "chunk_scores",
926 + "companies",
927 + "embeddings_metadata",
928 + "filings",
929 + "filing_chunks",
930 + "filing_scores",
931 + "filing_sections",
932 + "processing_log",
933 + "quarterly_final",
934 + "quarterly_panel"
935 + ],
936 + "temp_ok": true
937 +}
\ No newline at end of file
938
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere