spb/company-atlas

1"""Typed contracts shared by connectors, the pipeline and the intelligence layer.23    FetchResult (fetch.py)  →  Extraction (this module)  →  snapshot row + blocks object  →  BlockDiff / StructuredDelta  →  changes row4                                                                                                                             ↓5                                                                                      events service reads changes.structured_delta67Everything is plain dataclasses / pydantic so it serialises to JSON for the `snapshots.extracted` and `changes.structured_delta`8columns. Never put raw HTML in these — raw bytes live in the object store, referenced by hash.9"""10from __future__ import annotations1112from dataclasses import asdict, dataclass, field13from datetime import datetime14from typing import Any, Literal1516from companyatlas.taxonomy import Surface1718BlockKind = Literal["header", "hero", "nav", "section", "heading", "paragraph", "list", "table", "product_card", "pricing_plan", "job_listing",19                    "person", "location", "news_item", "faq", "footer", "code", "quote", "other"]202122@dataclass(slots=True)23class Block:24    """A semantic block of a page (spec §19). `key` is the stable identity used for added/removed/modified/moved diffing:25    it is derived from the block kind + heading path + a fuzzy content fingerprint, never from DOM position alone."""26    key: str27    kind: str28    text: str29    path: str = ""                    # heading breadcrumb, e.g. "Pricing > Pro"30    hash: str = ""                    # sha256 of normalized text (exact)31    simhash: int = 0                  # 64-bit near-duplicate fingerprint32    weight: float = 1.0               # importance weight (hero/pricing/job > footer/nav)33    order: int = 034    attrs: dict[str, Any] = field(default_factory=dict)3536    def to_json(self) -> dict[str, Any]:37        return asdict(self)383940@dataclass(slots=True)41class ExtractedJob:42    title: str43    url: str | None = None44    external_id: str | None = None45    department: str | None = None46    team: str | None = None47    location_text: str | None = None48    city: str | None = None49    region: str | None = None50    country: str | None = None          # ISO-2 when confidently derivable, else None (never guess)51    remote: bool | None = None52    employment_type: str | None = None53    seniority: str | None = None54    skills: list[str] = field(default_factory=list)55    salary_min: float | None = None56    salary_max: float | None = None57    salary_currency: str | None = None58    salary_period: str | None = None59    posted_at: datetime | None = None60    description_hash: str | None = None61    raw: dict[str, Any] = field(default_factory=dict)626364@dataclass(slots=True)65class ExtractedPerson:66    name: str67    title: str | None = None68    role_category: str | None = None    # ceo | cfo | cto | coo | founder | president | chair | board | vp | head | other69    is_executive: bool = False70    url: str | None = None717273@dataclass(slots=True)74class ExtractedProduct:75    name: str76    url: str | None = None77    category: str | None = None78    description: str | None = None798081@dataclass(slots=True)82class ExtractedPlan:83    plan_name: str84    price: float | None = None85    price_text: str | None = None86    currency: str | None = None87    billing_period: str | None = None   # month | year | one_time | usage | contact88    unit: str | None = None89    features: list[str] = field(default_factory=list)90    contact_sales: bool = False919293@dataclass(slots=True)94class ExtractedLocation:95    name: str96    kind: str = "office"                # headquarters | office | store | factory | warehouse | lab | data_center | other97    city: str | None = None98    region: str | None = None99    country: str | None = None100    address_text: str | None = None     # only if the page states it; never invented101102103@dataclass(slots=True)104class ExtractedNewsItem:105    title: str106    url: str107    published_at: datetime | None = None108    summary: str | None = None109    category: str | None = None         # press | blog | changelog | research | ir | other110    language: str | None = None111112113@dataclass(slots=True)114class DiscoveredUrl:115    url: str116    surface: Surface117    confidence: float118    anchor: str | None = None119    method: str = "nav"                 # nav | sitemap | robots | pattern | ats | feed | link | jsonld120121122@dataclass(slots=True)123class Extraction:124    """Output of `Connector.extract`. `text` is the normalized main-content text (what gets diffed at the text level),125    `blocks` the semantic blocks (block-level diff), the typed lists feed entity tables and structured deltas."""126    text: str127    blocks: list[Block]128    title: str | None = None129    language: str | None = None130    meta: dict[str, Any] = field(default_factory=dict)          # description, og tags, canonical, generator, jsonld types…131    jobs: list[ExtractedJob] = field(default_factory=list)132    people: list[ExtractedPerson] = field(default_factory=list)133    products: list[ExtractedProduct] = field(default_factory=list)134    plans: list[ExtractedPlan] = field(default_factory=list)135    locations: list[ExtractedLocation] = field(default_factory=list)136    news: list[ExtractedNewsItem] = field(default_factory=list)137    discovered: list[DiscoveredUrl] = field(default_factory=list)138    structured_hash: str = ""                                   # hash of the structured payload (typed lists) — set by the pipeline if empty139    normalized_hash: str = ""                                   # hash of `text` — set by the pipeline if empty140141    def summary(self) -> dict[str, Any]:142        return {"job_count": len(self.jobs), "people_count": len(self.people), "product_count": len(self.products), "plan_count": len(self.plans),143                "location_count": len(self.locations), "news_count": len(self.news), "block_count": len(self.blocks), "text_length": len(self.text),144                "discovered_count": len(self.discovered)}145146    def structured_payload(self) -> dict[str, Any]:147        return {"jobs": [asdict(j) for j in self.jobs], "people": [asdict(p) for p in self.people], "products": [asdict(p) for p in self.products],148                "plans": [asdict(p) for p in self.plans], "locations": [asdict(loc) for loc in self.locations], "news": [asdict(n) for n in self.news],149                "meta": self.meta}150151152@dataclass(slots=True)153class BlockDelta:154    key: str155    kind: str156    path: str157    before: str | None158    after: str | None159    weight: float160    similarity: float | None = None     # for modified blocks161162163@dataclass(slots=True)164class BlockDiff:165    """Result of comparing two block lists (spec §19–20). `significance` ∈ [0,1] is computed by the diff engine from the weighted166    share of changed content, page importance and the typed deltas; the pipeline maps it to a ChangeKind."""167    added: list[BlockDelta] = field(default_factory=list)168    removed: list[BlockDelta] = field(default_factory=list)169    modified: list[BlockDelta] = field(default_factory=list)170    moved: list[str] = field(default_factory=list)171    text_delta_ratio: float = 0.0172    similarity: float = 1.0173    significance: float = 0.0174    reasons: list[str] = field(default_factory=list)175176    @property177    def is_empty(self) -> bool:178        return not (self.added or self.removed or self.modified)179180    def to_json(self, *, limit: int = 40) -> dict[str, Any]:181        def _cut(items: list[BlockDelta]) -> list[dict[str, Any]]:182            out = []183            for d in items[:limit]:184                out.append({"key": d.key, "kind": d.kind, "path": d.path, "before": (d.before or "")[:600] or None, "after": (d.after or "")[:600] or None,185                            "weight": d.weight, "similarity": d.similarity})186            return out187        return {"added": _cut(self.added), "removed": _cut(self.removed), "modified": _cut(self.modified), "moved": self.moved[:limit],188                "counts": {"added": len(self.added), "removed": len(self.removed), "modified": len(self.modified), "moved": len(self.moved)},189                "text_delta_ratio": round(self.text_delta_ratio, 4), "similarity": round(self.similarity, 4), "reasons": self.reasons}190191192# StructuredDelta — JSON stored in `changes.structured_delta`, produced by the pipeline when it reconciles typed extractions193# with the entity tables. Shape (all keys optional, lists bounded to 200 items):194#   {195#     "jobs":      {"added": [ {title, url, location_text, country, remote, department, is_ai} ], "removed": [...], "open_before": n, "open_after": n},196#     "people":    {"added": [ {name, title, role_category} ], "removed": [...], "title_changed": [ {name, before, after} ]},197#     "products":  {"added": [ {name, url} ], "removed": [...]},198#     "plans":     {"added": [ {plan_name, price, currency, billing_period} ], "removed": [...],199#                   "price_changed": [ {plan_name, before, after, currency, billing_period, pct} ]},200#     "locations": {"added": [ {name, city, country, kind} ], "removed": [...], "new_countries": ["JP"]},201#     "news":      {"added": [ {title, url, published_at, category} ]},202#     "meta":      {"title_changed": {before, after}, "description_changed": bool, "language": "en"}203#   }204StructuredDelta = dict[str, Any]205206207__all__ = ["Block", "BlockDelta", "BlockDiff", "BlockKind", "DiscoveredUrl", "ExtractedJob", "ExtractedLocation", "ExtractedNewsItem", "ExtractedPerson",208           "ExtractedPlan", "ExtractedProduct", "Extraction", "StructuredDelta"]209