spb/uqo-research-db

1# ============================================================2# SPID — deterministic rule-based extractor3#4# High-precision baseline that needs NO LLM.  Scans section text5# sentence-by-sentence for project signals, classifies the type,6# and pulls money / location / tech / year / status.7# ============================================================8from __future__ import annotations910import re1112from spid import taxonomy as tx13from spid.extract.base import ProjectMention1415# split into sentence-ish units; keep it cheap for 100k+ sections16_SENT_SPLIT = re.compile(r"(?<=[.;:!?])\s+(?=[A-Z(\"$])")17_WS = re.compile(r"\s+")18# generic "project-ish" gate so we skip boilerplate sentences fast19_PROJECT_GATE = re.compile(20    r"\b(project|initiative|program|invest|expan|construct|build|launch|develop|"21    r"deploy|implement|acqui|partner|facilit|capacity|capital|migrat|transform|"22    r"pipeline|plant|data ?cent|automat|sustainab|renewable|restructur)\w*",23    re.IGNORECASE,24)25_NAMED_PROJECT = re.compile(26    r"\b((?:Project|Program|Initiative)\s+[A-Z][A-Za-z0-9\-]+"27    r"|[A-Z][A-Za-z0-9\-]+\s+(?:Project|Program|Initiative))\b"28)293031def _clean(s: str) -> str:32    return _WS.sub(" ", s).strip()333435class RuleExtractor:36    name = "rules"3738    def extract(self, *, text: str, accession_number: str, cik: str,39                section_id: str, section_name: str) -> list[ProjectMention]:40        if not text:41            return []42        text = text[:600_000]43        out: list[ProjectMention] = []44        seen: set[str] = set()4546        for raw_sent in _SENT_SPLIT.split(text):47            sent = _clean(raw_sent)48            if not (20 <= len(sent) <= 700):49                continue50            if not _PROJECT_GATE.search(sent):51                continue5253            low = sent.lower()54            matched_types: list[str] = []55            for ptype, regexes in tx.TYPE_SIGNALS.items():56                if any(rx.search(low) for rx in regexes):57                    matched_types.append(ptype)58            if not matched_types:59                continue6061            amount, amount_raw = tx.extract_money(sent)62            locations = tx.extract_locations(sent)63            techs = tx.extract_technologies(sent)64            years = tx.extract_years(sent)65            status = tx.detect_status(sent)66            nm = _NAMED_PROJECT.search(sent)6768            # keep at most the 2 strongest types for one sentence69            for ptype in matched_types[:2]:70                anchor = f"{ptype}:{(locations[0] if locations else '')}:{int(amount or 0)}"71                if anchor in seen:72                    continue73                seen.add(anchor)7475                conf = self._confidence(ptype, amount, locations, years, status, nm, low)76                if conf < 0.30:77                    continue7879                pname = nm.group(1) if nm else self._synth_name(ptype, locations, techs)80                out.append(ProjectMention(81                    mention_id=ProjectMention.make_id(accession_number, section_id, ptype, anchor),82                    accession_number=accession_number, cik=cik,83                    section_id=section_id, section_name=section_name,84                    project_name=pname, project_type=ptype,85                    description=sent[:500],86                    amount_usd=amount, amount_raw=amount_raw,87                    locations=locations, technologies=techs, years=years,88                    status=status, confidence=round(conf, 3), backend=self.name,89                ))90        return out9192    @staticmethod93    def _synth_name(ptype: str, locations: list[str], techs: list[str]) -> str:94        label = tx.TYPE_LABELS.get(ptype, ptype)95        if locations:96            return f"{label} — {locations[0]}"97        if techs:98            return f"{label} ({techs[0]})"99        return label100101    @staticmethod102    def _confidence(ptype, amount, locations, years, status, nm, low) -> float:103        c = 0.40104        if nm:105            c += 0.20106        if amount:107            c += 0.18108        if locations:109            c += 0.10110        if years:111            c += 0.06112        if status in ("planned", "in_progress", "completed"):113            c += 0.08114        # strong, specific phrasings115        if re.search(r"\b(announced|committed|invest\w+|will (build|open|invest))\b", low):116            c += 0.06117        # de-weight vague risk-factor / forward-looking hedges118        if re.search(r"\b(may|might|could|potential\w*|if we|uncertain)\b", low):119            c -= 0.08120        return max(0.0, min(1.0, c))121