# ============================================================ # SPID — deterministic rule-based extractor # # High-precision baseline that needs NO LLM. Scans section text # sentence-by-sentence for project signals, classifies the type, # and pulls money / location / tech / year / status. # ============================================================ from __future__ import annotations import re from spid import taxonomy as tx from spid.extract.base import ProjectMention # split into sentence-ish units; keep it cheap for 100k+ sections _SENT_SPLIT = re.compile(r"(?<=[.;:!?])\s+(?=[A-Z(\"$])") _WS = re.compile(r"\s+") # generic "project-ish" gate so we skip boilerplate sentences fast _PROJECT_GATE = re.compile( r"\b(project|initiative|program|invest|expan|construct|build|launch|develop|" r"deploy|implement|acqui|partner|facilit|capacity|capital|migrat|transform|" r"pipeline|plant|data ?cent|automat|sustainab|renewable|restructur)\w*", re.IGNORECASE, ) _NAMED_PROJECT = re.compile( r"\b((?:Project|Program|Initiative)\s+[A-Z][A-Za-z0-9\-]+" r"|[A-Z][A-Za-z0-9\-]+\s+(?:Project|Program|Initiative))\b" ) def _clean(s: str) -> str: return _WS.sub(" ", s).strip() class RuleExtractor: name = "rules" def extract(self, *, text: str, accession_number: str, cik: str, section_id: str, section_name: str) -> list[ProjectMention]: if not text: return [] text = text[:600_000] out: list[ProjectMention] = [] seen: set[str] = set() for raw_sent in _SENT_SPLIT.split(text): sent = _clean(raw_sent) if not (20 <= len(sent) <= 700): continue if not _PROJECT_GATE.search(sent): continue low = sent.lower() matched_types: list[str] = [] for ptype, regexes in tx.TYPE_SIGNALS.items(): if any(rx.search(low) for rx in regexes): matched_types.append(ptype) if not matched_types: continue amount, amount_raw = tx.extract_money(sent) locations = tx.extract_locations(sent) techs = tx.extract_technologies(sent) years = tx.extract_years(sent) status = tx.detect_status(sent) nm = _NAMED_PROJECT.search(sent) # keep at most the 2 strongest types for one sentence for ptype in matched_types[:2]: anchor = f"{ptype}:{(locations[0] if locations else '')}:{int(amount or 0)}" if anchor in seen: continue seen.add(anchor) conf = self._confidence(ptype, amount, locations, years, status, nm, low) if conf < 0.30: continue pname = nm.group(1) if nm else self._synth_name(ptype, locations, techs) out.append(ProjectMention( mention_id=ProjectMention.make_id(accession_number, section_id, ptype, anchor), accession_number=accession_number, cik=cik, section_id=section_id, section_name=section_name, project_name=pname, project_type=ptype, description=sent[:500], amount_usd=amount, amount_raw=amount_raw, locations=locations, technologies=techs, years=years, status=status, confidence=round(conf, 3), backend=self.name, )) return out @staticmethod def _synth_name(ptype: str, locations: list[str], techs: list[str]) -> str: label = tx.TYPE_LABELS.get(ptype, ptype) if locations: return f"{label} — {locations[0]}" if techs: return f"{label} ({techs[0]})" return label @staticmethod def _confidence(ptype, amount, locations, years, status, nm, low) -> float: c = 0.40 if nm: c += 0.20 if amount: c += 0.18 if locations: c += 0.10 if years: c += 0.06 if status in ("planned", "in_progress", "completed"): c += 0.08 # strong, specific phrasings if re.search(r"\b(announced|committed|invest\w+|will (build|open|invest))\b", low): c += 0.06 # de-weight vague risk-factor / forward-looking hedges if re.search(r"\b(may|might|could|potential\w*|if we|uncertain)\b", low): c -= 0.08 return max(0.0, min(1.0, c))